import { TransformChunkSizes } from "./transform-chunk-sizes.ts";import * as errors from "./errors.ts";import { isValidBucketName, isValidObjectName, isValidPort, makeDateLong, sanitizeETag, sha256digestHex,} from "./helpers.ts";import { ObjectUploader } from "./object-uploader.ts";import { presignV4, signV4 } from "./signing.ts";import { parse as parseXML } from "./xml-parser.ts";
export interface ClientOptions { endPoint: string; accessKey?: string; secretKey?: string; sessionToken?: string; useSSL?: boolean | undefined; port?: number | undefined; bucket?: string; region: string; pathStyle?: boolean | undefined;}
const metadataKeys = [ "Content-Type", "Cache-Control", "Content-Disposition", "Content-Encoding", "Content-Language", "Expires", "x-amz-acl", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-server-side-encryption", "x-amz-storage-class", "x-amz-website-redirect-location", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "x-amz-server-side-encryption-aws-kms-key-id", "x-amz-server-side-encryption-context", "x-amz-server-side-encryption-bucket-key-enabled", "x-amz-request-payer", "x-amz-tagging", "x-amz-object-lock-mode", "x-amz-object-lock-retain-until-date", "x-amz-object-lock-legal-hold", "x-amz-expected-bucket-owner",] as const;
export type ObjectMetadata = { [K in typeof metadataKeys[number]]?: string } & { [key: string]: string };
export interface ResponseOverrideParams { "response-content-type"?: string; "response-content-language"?: string; "response-expires"?: string; "response-cache-control"?: string; "response-content-disposition"?: string; "response-content-encoding"?: string;}
export interface UploadedObjectInfo { etag: string; versionId: string | null;}
export interface CopiedObjectInfo extends UploadedObjectInfo { lastModified: Date; copySourceVersionId: string | null;}
export interface S3Object { type: "Object"; key: string; lastModified: Date; etag: string; size: number;}export interface CommonPrefix { type: "CommonPrefix"; prefix: string;}
export interface ObjectStatus extends S3Object { versionId: string | null; metadata: ObjectMetadata;}
const minimumPartSize = 5 * 1024 * 1024;const maximumPartSize = 5 * 1024 * 1024 * 1024;const maxObjectSize = 5 * 1024 * 1024 * 1024 * 1024;
export class Client { readonly host: string; readonly port: number; readonly protocol: "https:" | "http:"; readonly accessKey?: string; readonly #secretKey: string; readonly sessionToken?: string; readonly defaultBucket: string | undefined; readonly region: string; readonly userAgent = "deno-s3-lite-client"; readonly pathStyle: boolean;
constructor(params: ClientOptions) { if (params.useSSL === undefined) { params.useSSL = true; } if ( typeof params.endPoint !== "string" || params.endPoint.length === 0 || params.endPoint.indexOf("/") !== -1 ) { throw new errors.InvalidEndpointError( `Invalid endPoint : ${params.endPoint}`, ); } if (params.port !== undefined && !isValidPort(params.port)) { throw new errors.InvalidArgumentError(`Invalid port : ${params.port}`); } if (params.accessKey && !params.secretKey) { throw new errors.InvalidArgumentError(`If specifying access key, secret key must also be provided.`); } if (params.accessKey && params.accessKey.startsWith("ASIA") && !params.sessionToken) { throw new errors.InvalidArgumentError(`If specifying temporary access key, session token must also be provided.`); }
const defaultPort = params.useSSL ? 443 : 80; this.port = params.port ?? defaultPort; this.host = params.endPoint.toLowerCase() + (this.port !== defaultPort ? `:${params.port}` : ""); this.protocol = params.useSSL ? "https:" : "http:"; this.accessKey = params.accessKey; this.#secretKey = params.secretKey ?? ""; this.sessionToken = params.sessionToken; this.pathStyle = params.pathStyle ?? true; this.defaultBucket = params.bucket; this.region = params.region; }
protected getBucketName(options: undefined | { bucketName?: string }) { const bucketName = options?.bucketName ?? this.defaultBucket; if (bucketName === undefined || !isValidBucketName(bucketName)) { throw new errors.InvalidBucketNameError( `Invalid bucket name: ${bucketName}`, ); } return bucketName; }
private buildRequestOptions(options: { objectName: string; bucketName?: string; headers?: Headers; query?: string | Record<string, string>; }): { headers: Headers; host: string; path: string; } { const bucketName = this.getBucketName(options); const host = this.pathStyle ? this.host : `${bucketName}.${this.host}`; const headers = options.headers ?? new Headers(); headers.set("host", host); const queryAsString = typeof options.query === "object" ? new URLSearchParams(options.query).toString().replace("+", "%20") : (options.query); const path = (this.pathStyle ? `/${bucketName}/${options.objectName}` : `/${options.objectName}`) + (queryAsString ? `?${queryAsString}` : ""); return { headers, host, path }; }
public async makeRequest({ method, payload, ...options }: { method: "POST" | "GET" | "PUT" | "DELETE" | string; headers?: Headers; query?: string | Record<string, string>; objectName: string; bucketName?: string; statusCode?: number; payload?: Uint8Array | string; returnBody?: boolean; }): Promise<Response> { const date = new Date(); const { headers, host, path } = this.buildRequestOptions(options); const statusCode = options.statusCode ?? 200;
if ( method === "POST" || method === "PUT" || method === "DELETE" ) { if (payload === undefined) { payload = new Uint8Array(); } else if (typeof payload === "string") { payload = new TextEncoder().encode(payload); } headers.set("Content-Length", String(payload.length)); } else if (payload) { throw new Error(`Unexpected payload on ${method} request.`); } const sha256sum = await sha256digestHex(payload ?? new Uint8Array()); headers.set("x-amz-date", makeDateLong(date)); headers.set("x-amz-content-sha256", sha256sum); if (this.accessKey) { if (this.sessionToken) { headers.set("x-amz-security-token", this.sessionToken); } headers.set( "authorization", await signV4({ headers, method, path, accessKey: this.accessKey, secretKey: this.#secretKey, region: this.region, date, }), ); }
const fullUrl = `${this.protocol}//${host}${path}`;
const response = await fetch(fullUrl, { method, headers, body: payload, });
if (response.status !== statusCode) { if (response.status >= 400) { const error = await errors.parseServerError(response); throw error; } else if (response.status === 301) { throw new errors.ServerError( response.status, "UnexpectedRedirect", `The server unexpectedly returned a redirect response. With AWS S3, this usually means you need to use a ` + `region-specific endpoint like "s3.us-west-2.amazonaws.com" instead of "s3.amazonaws.com"`, ); } throw new errors.ServerError( response.status, "UnexpectedStatusCode", `Unexpected response code from the server (expected ${statusCode}, got ${response.status} ${response.statusText}).`, ); } if (!options.returnBody) { await response.body?.getReader().read(); } return response; }
async deleteObject( objectName: string, options: { bucketName?: string; versionId?: string; governanceBypass?: boolean } = {}, ) { const bucketName = this.getBucketName(options); if (!isValidObjectName(objectName)) { throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`); }
const query: Record<string, string> = options.versionId ? { versionId: options.versionId } : {}; const headers = new Headers(); if (options.governanceBypass) { headers.set("X-Amz-Bypass-Governance-Retention", "true"); }
await this.makeRequest({ method: "DELETE", bucketName, objectName, headers, query, statusCode: 204, }); }
public async exists(objectName: string, options?: { bucketName?: string; versionId?: string }): Promise<boolean> { try { await this.statObject(objectName, options); return true; } catch (err: unknown) { if (err instanceof errors.ServerError && err.statusCode === 404) { return false; } throw err; } }
public getObject( objectName: string, options?: { metadata?: ObjectMetadata; bucketName?: string; versionId?: string; responseParams?: ResponseOverrideParams; }, ): Promise<Response> { return this.getPartialObject(objectName, { ...options, offset: 0, length: 0 }); }
public async getPartialObject( objectName: string, { offset, length, ...options }: { offset: number; length: number; metadata?: ObjectMetadata; bucketName?: string; versionId?: string; responseParams?: ResponseOverrideParams; }, ): Promise<Response> { const bucketName = this.getBucketName(options); if (!isValidObjectName(objectName)) { throw new errors.InvalidObjectNameError( `Invalid object name: ${objectName}`, ); }
const headers = new Headers(Object.entries(options.metadata ?? {})); let statusCode = 200; if (offset || length) { let range = ""; if (offset) { range = `bytes=${+offset}-`; } else { range = "bytes=0-"; offset = 0; } if (length) { range += `${(+length + offset) - 1}`; } headers.set("Range", range); statusCode = 206; }
const query: Record<string, string> = { ...options.responseParams, ...(options.versionId ? { versionId: options.versionId } : {}), }; return await this.makeRequest({ method: "GET", bucketName, objectName, headers, query, statusCode, returnBody: true, }); }
getPresignedUrl( method: "GET" | "PUT" | "HEAD" | "DELETE", objectName: string, options: { bucketName?: string; parameters?: Record<string, string>; expirySeconds?: number; requestDate?: Date } = {}, ): Promise<string> { if (!this.accessKey) { throw new errors.AccessKeyRequiredError( `Presigned ${method} URLs cannot be generated for anonymous requests. Specify an accessKey and secretKey.`, ); } if (!isValidObjectName(objectName)) { throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`); } const { headers, path } = this.buildRequestOptions({ objectName, bucketName: options.bucketName, query: options.parameters, }); const requestDate = options.requestDate ?? new Date(); const expirySeconds = options.expirySeconds ?? 24 * 60 * 60 * 7;
return presignV4({ protocol: this.protocol, headers, method, path, accessKey: this.accessKey, secretKey: this.#secretKey, region: this.region, date: requestDate, expirySeconds, }); }
presignedGetObject( objectName: string, options: { bucketName?: string; versionId?: string; responseParams?: ResponseOverrideParams; expirySeconds?: number; requestDate?: Date; } = {}, ) { const { versionId, responseParams, ...otherOptions } = options; const parameters: Record<string, string> = { ...responseParams, ...(versionId ? { versionId } : {}), }; return this.getPresignedUrl("GET", objectName, { parameters, ...otherOptions }); }
public async *listObjects( options: { prefix?: string; bucketName?: string; maxResults?: number; pageSize?: number; } = {}, ): AsyncGenerator<S3Object, void, undefined> { for await (const result of this.listObjectsGrouped({ ...options, delimiter: "" })) { if (result.type === "Object") { yield result; } else { throw new Error(`Unexpected result from listObjectsGrouped(): ${result}`); } } }
public async *listObjectsGrouped( options: { delimiter: string; prefix?: string; bucketName?: string; maxResults?: number; pageSize?: number; }, ): AsyncGenerator<S3Object | CommonPrefix, void, undefined> { const bucketName = this.getBucketName(options); let continuationToken = ""; const pageSize = options.pageSize ?? 1_000; if (pageSize < 1 || pageSize > 1_000) { throw new errors.InvalidArgumentError("pageSize must be between 1 and 1,000."); } let resultCount = 0;
while (true) { const maxKeys = options.maxResults ? Math.min(pageSize, options.maxResults - resultCount) : pageSize; if (maxKeys === 0) { return; } const pageResponse = await this.makeRequest({ method: "GET", bucketName, objectName: "", query: { "list-type": "2", prefix: options.prefix ?? "", delimiter: options.delimiter, "max-keys": String(maxKeys), ...(continuationToken ? { "continuation-token": continuationToken } : {}), }, returnBody: true, }); const responseText = await pageResponse.text(); const root = parseXML(responseText).root; if (!root || root.name !== "ListBucketResult") { throw new Error(`Unexpected response: ${responseText}`); } const prefixElements = root.children .filter((c) => c.name === "CommonPrefixes") .flatMap((c) => c.children); const toYield: Array<S3Object | CommonPrefix> = []; for (const prefixElement of prefixElements) { toYield.push({ type: "CommonPrefix", prefix: prefixElement.content ?? "", }); resultCount++; } for (const objectElement of root.children.filter((c) => c.name === "Contents")) { toYield.push({ type: "Object", key: objectElement.children.find((c) => c.name === "Key")?.content ?? "", etag: sanitizeETag(objectElement.children.find((c) => c.name === "ETag")?.content ?? ""), size: parseInt(objectElement.children.find((c) => c.name === "Size")?.content ?? "", 10), lastModified: new Date(objectElement.children.find((c) => c.name === "LastModified")?.content ?? "invalid"), }); resultCount++; } toYield.sort((a, b) => { const aStr = a.type === "Object" ? a.key : a.prefix; const bStr = b.type === "Object" ? b.key : b.prefix; return aStr > bStr ? 1 : aStr < bStr ? -1 : 0; }); for (const entry of toYield) { yield entry; } const isTruncated = root.children.find((c) => c.name === "IsTruncated")?.content === "true"; if (isTruncated) { const nextContinuationToken = root.children.find((c) => c.name === "NextContinuationToken")?.content; if (!nextContinuationToken) { throw new Error("Unexpectedly missing continuation token, but server said there are more results."); } continuationToken = nextContinuationToken; } else { return; } } }
async putObject( objectName: string, streamOrData: ReadableStream<Uint8Array> | Uint8Array | string, options?: { metadata?: ObjectMetadata; size?: number; bucketName?: string; partSize?: number; }, ): Promise<UploadedObjectInfo> { const bucketName = this.getBucketName(options); if (!isValidObjectName(objectName)) { throw new errors.InvalidObjectNameError( `Invalid object name: ${objectName}`, ); }
let size: number | undefined; let stream: ReadableStream<Uint8Array>; if (typeof streamOrData === "string") { const binaryData = new TextEncoder().encode(streamOrData); stream = ReadableStream.from([binaryData]); size = binaryData.length; } else if (streamOrData instanceof Uint8Array) { stream = ReadableStream.from([streamOrData]); size = streamOrData.byteLength; } else if (streamOrData instanceof ReadableStream) { stream = streamOrData; } else { throw new errors.InvalidArgumentError( `Invalid stream/data type provided.`, ); }
if (options?.size !== undefined) { if (size !== undefined && options?.size !== size) { throw new errors.InvalidArgumentError( `size was specified (${options.size}) but doesn't match auto-detected size (${size}).`, ); } if (typeof options.size !== "number" || options.size < 0 || isNaN(options.size)) { throw new errors.InvalidArgumentError( `invalid size specified: ${options.size}`, ); } else { size = options.size; } }
const partSize = options?.partSize ?? this.calculatePartSize(size); if (partSize < minimumPartSize) { throw new errors.InvalidArgumentError(`Part size should be greater than 5MB`); } else if (partSize > maximumPartSize) { throw new errors.InvalidArgumentError(`Part size should be less than 6MB`); }
const chunker = new TransformChunkSizes(partSize);
const uploader = new ObjectUploader({ client: this, bucketName, objectName, partSize, metadata: options?.metadata ?? {}, }); await stream.pipeThrough(chunker).pipeTo(uploader); return uploader.getResult(); }
protected calculatePartSize(size: number | undefined) { if (size === undefined) { size = maxObjectSize; } if (size > maxObjectSize) { throw new TypeError(`size should not be more than ${maxObjectSize}`); } let partSize = 64 * 1024 * 1024; while (true) { if ((partSize * 10_000) > size) { return partSize; } partSize += 16 * 1024 * 1024; } }
public async statObject( objectName: string, options?: { bucketName?: string; versionId?: string }, ): Promise<ObjectStatus> { const bucketName = this.getBucketName(options); if (!isValidObjectName(objectName)) { throw new errors.InvalidObjectNameError( `Invalid object name: ${objectName}`, ); } const query: Record<string, string> = {}; if (options?.versionId) { query.versionId = options.versionId; } const response = await this.makeRequest({ method: "HEAD", bucketName, objectName, query, });
const metadata: ObjectMetadata = {}; for (const header of metadataKeys) { if (response.headers.has(header)) { metadata[header] = response.headers.get(header) as string; } } response.headers.forEach((_value, key) => { if (key.startsWith("x-amz-meta-")) { metadata[key] = response.headers.get(key) as string; } });
return { type: "Object", key: objectName, size: parseInt(response.headers.get("content-length") ?? "", 10), metadata, lastModified: new Date(response.headers.get("Last-Modified") ?? "error: missing last modified"), versionId: response.headers.get("x-amz-version-id") || null, etag: sanitizeETag(response.headers.get("ETag") ?? ""), }; }
public async copyObject( source: { sourceBucketName?: string; sourceKey: string; sourceVersionId?: string }, objectName: string, options?: { bucketName?: string }, ): Promise<CopiedObjectInfo> { const bucketName = this.getBucketName(options); const sourceBucketName = source.sourceBucketName ?? bucketName; if (!isValidObjectName(objectName)) { throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`); }
let xAmzCopySource = `${sourceBucketName}/${source.sourceKey}`; if (source.sourceVersionId) xAmzCopySource += `?versionId=${source.sourceVersionId}`;
const response = await this.makeRequest({ method: "PUT", bucketName, objectName, headers: new Headers({ "x-amz-copy-source": xAmzCopySource }), returnBody: true, });
const responseText = await response.text(); const root = parseXML(responseText).root; if (!root || root.name !== "CopyObjectResult") { throw new Error(`Unexpected response: ${responseText}`); } const etagString = root.children.find((c) => c.name === "ETag")?.content ?? ""; const lastModifiedString = root.children.find((c) => c.name === "LastModified")?.content; if (lastModifiedString === undefined) { throw new Error("Unable to find <LastModified>...</LastModified> from the server."); }
return { copySourceVersionId: response.headers.get("x-amz-copy-source-version-id") || null, etag: sanitizeETag(etagString), lastModified: new Date(lastModifiedString), versionId: response.headers.get("x-amz-version-id") || null, }; }
public async bucketExists(bucketName: string): Promise<boolean> { try { const objects = this.listObjects({ bucketName }); await objects.next(); return true; } catch (err: unknown) { if (err instanceof errors.ServerError && err.statusCode === 404) { return false; } throw err; } }
public async makeBucket(bucketName: string): Promise<void> { await this.makeRequest({ method: "PUT", bucketName: this.getBucketName({ bucketName }), objectName: "", statusCode: 200, }); }
public async removeBucket(bucketName: string): Promise<void> { await this.makeRequest({ method: "DELETE", bucketName: this.getBucketName({ bucketName }), objectName: "", statusCode: 204, }); }}