Skip to content

Commit

Permalink
feat: Support more JavaScript runtimes (PR #31)
Browse files Browse the repository at this point in the history
  • Loading branch information
bradenmacdonald authored Mar 3, 2024
2 parents b8af26b + 96e0644 commit 1e89ae3
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 8 deletions.
19 changes: 19 additions & 0 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
name: Publish to JSR
on:
push:
tags:
- '*' # Publish every time a tag is pushed (unless it contains '/')

jobs:
publish:
runs-on: ubuntu-latest

permissions:
contents: read
id-token: write

steps:
- uses: actions/checkout@v4

- name: Publish package
run: npx jsr publish
18 changes: 18 additions & 0 deletions .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,21 @@ jobs:

- name: Run integration tests
run: deno test --allow-net integration.ts

test-bun:
runs-on: ubuntu-latest
steps:
- name: Setup repo
uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v1
with:
bun-version: latest
- name: Start MinIO for integration tests
run: docker run --name minio --detach -e MINIO_ROOT_USER=AKIA_DEV -e MINIO_ROOT_PASSWORD=secretkey -e MINIO_REGION_NAME=dev-region -p 9000:9000 -p 9001:9001 --entrypoint /bin/sh minio/minio:RELEASE.2021-10-23T03-28-24Z -c 'mkdir -p /data/dev-bucket && minio server --console-address ":9001" /data'
# TODO: can we get jsr to load the dependency versions from deno.jsonc?
- name: Install dependencies
run: bunx jsr add @std/io @std/assert
- name: Convert integration test from Deno to Bun test runner
run: '(echo -e ''import { test } from "bun:test";\nconst Deno = { test: ({fn}: {fn: () => void, name: string}) => test(fn) };''; cat integration.ts ) > integration-bun.ts'
- name: Run integration tests with bun
run: bun test ./integration-bun.ts
28 changes: 25 additions & 3 deletions client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,9 @@ const maximumPartSize = 5 * 1024 * 1024 * 1024;
/** The maximum allowed object size for multi-part uploads. https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html */
const maxObjectSize = 5 * 1024 * 1024 * 1024 * 1024;

/**
* Client for connecting to S3-compatible object storage services.
*/
export class Client {
readonly host: string;
readonly port: number;
Expand Down Expand Up @@ -173,6 +176,7 @@ export class Client {
this.region = params.region;
}

/** Internal helper method to figure out which bucket name to use for a request */
protected getBucketName(options: undefined | { bucketName?: string }): string {
const bucketName = options?.bucketName ?? this.defaultBucket;
if (bucketName === undefined || !isValidBucketName(bucketName)) {
Expand All @@ -185,7 +189,6 @@ export class Client {

/**
* Common code used for both "normal" requests and presigned UTL requests
* @param param0
*/
private buildRequestOptions(options: {
objectName: string;
Expand Down Expand Up @@ -620,6 +623,9 @@ export class Client {
}
}

/**
* Upload an object
*/
async putObject(
objectName: string,
streamOrData: ReadableStream<Uint8Array> | Uint8Array | string,
Expand Down Expand Up @@ -648,10 +654,23 @@ export class Client {
if (typeof streamOrData === "string") {
// Convert to binary using UTF-8
const binaryData = new TextEncoder().encode(streamOrData);
stream = ReadableStream.from([binaryData]);
if (typeof ReadableStream.from !== "undefined") {
stream = ReadableStream.from([binaryData]);
} else {
// ReadableStream.from is not yet supported by some runtimes :/
// https://github.com/oven-sh/bun/issues/3700
// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream/from_static#browser_compatibility
// deno-fmt-ignore
stream = new ReadableStream({ start(c) { c.enqueue(binaryData); c.close(); } });
}
size = binaryData.length;
} else if (streamOrData instanceof Uint8Array) {
stream = ReadableStream.from([streamOrData]);
if (typeof ReadableStream.from !== "undefined") {
stream = ReadableStream.from([streamOrData]);
} else {
// deno-fmt-ignore
stream = new ReadableStream({ start(c) { c.enqueue(streamOrData); c.close(); } });
}
size = streamOrData.byteLength;
} else if (streamOrData instanceof ReadableStream) {
stream = streamOrData;
Expand Down Expand Up @@ -831,6 +850,7 @@ export class Client {
};
}

/** Check if a bucket exists */
public async bucketExists(bucketName: string): Promise<boolean> {
try {
const objects = this.listObjects({ bucketName });
Expand All @@ -845,6 +865,7 @@ export class Client {
}
}

/** Create a new bucket */
public async makeBucket(bucketName: string): Promise<void> {
await this.makeRequest({
method: "PUT",
Expand All @@ -854,6 +875,7 @@ export class Client {
});
}

/** Delete a bucket (must be empty) */
public async removeBucket(bucketName: string): Promise<void> {
await this.makeRequest({
method: "DELETE",
Expand Down
25 changes: 20 additions & 5 deletions integration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,26 @@ Deno.test({
name: "putObject() can stream a large file upload",
fn: async () => {
// First generate a 32MiB file in memory, 1 MiB at a time, as a stream
const dataStream = ReadableStream.from(async function* () {
for (let i = 0; i < 32; i++) {
yield new Uint8Array(1024 * 1024).fill(i % 256); // Yield 1MB of data
}
}());
let dataStream;
if (typeof ReadableStream.from !== "undefined") {
dataStream = ReadableStream.from(async function* () {
for (let i = 0; i < 32; i++) {
yield new Uint8Array(1024 * 1024).fill(i); // Yield 1MB of data
}
}());
} else {
// ReadableStream.from is not yet supported by some runtimes :/
// https://github.com/oven-sh/bun/issues/3700
// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream/from_static#browser_compatibility
let i = 0;
dataStream = new ReadableStream({
pull(controller) {
if (i < 32) {
controller.enqueue(new Uint8Array(1024 * 1024).fill(i++));
} else controller.close();
},
});
}

// Upload the 32MB stream data as 7 5MB parts. The client doesn't know in advance how big the stream is.
const key = "test-32m.dat";
Expand Down
9 changes: 9 additions & 0 deletions mod.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
/**
* @module
* A lightweight client for connecting to S3-compatible object storage services.
*/

export { Client as S3Client } from "./client.ts";

/**
* Namespace for all errors that can be thrown by S3Client
*/
export * as S3Errors from "./errors.ts";

0 comments on commit 1e89ae3

Please sign in to comment.