in my local datacenter we use S3 compatible storage. I wanted to create a k6 script which
However for this I needed "forcePathStyle: true" and this was not possible with the library.
Unfortunately I do not know anything about coding, programming and scripts. Fortunately I had AI access and so I created my k6 script with AI and I asked the AI to modify the library to work with my environment.
I want to share these results - maybe someone with knowledge can create PRs and add the changes to the library or at least use the modified library if needed.
import { parseHTML as Q } from "k6/html";
import p from "k6/http";
import E from "k6/crypto";
// --- Helper function for case-insensitive header extraction ---
function getHeader(headers, name) {
for (let k in headers) {
if (k.toLowerCase() === name.toLowerCase()) return headers[k];
}
return "";
}
// --- Helper class for endpoint parsing and manipulation ---
class Endpoint {
_protocol;
_hostname;
_port;
static DEFAULT_PROTOCOL = "https";
constructor(e) {
let s = !e.startsWith("http://") && !e.startsWith("https://") ? `${Endpoint.DEFAULT_PROTOCOL}://${e}` : e,
t = s.match(/^https?:/),
n = s.replace(/^https?:\/\//, ""),
[o] = n.split("/");
this._protocol = t ? t[0].slice(0, -1) : Endpoint.DEFAULT_PROTOCOL;
this._hostname = o.split(":")[0];
this._port = o.split(":")[1] ? parseInt(o.split(":")[1]) : void 0;
}
copy() { return new Endpoint(this.href); }
get host() { return this._port ? `${this._hostname}:${this._port}` : this._hostname; }
set host(e) { let [r, s] = e.split(":"); this._hostname = r, this._port = s ? parseInt(s) : void 0; }
get hostname() { return this._hostname; }
set hostname(e) { this._hostname = e; }
get href() { return `${this.protocol}://${this.host}`; }
set href(e) {
let r = e.match(/^https?:/),
s = e.replace(/^https?:\/\//, ""),
[t] = s.split("/");
this._protocol = r ? r[0].slice(0, -1) : Endpoint.DEFAULT_PROTOCOL;
this._hostname = t.split(":")[0];
this._port = t.split(":")[1] ? parseInt(t.split(":")[1]) : void 0;
}
get port() { return this._port; }
set port(e) { this._port = e; }
get protocol() { return this._protocol; }
set protocol(e) { this._protocol = e; }
}
// --- AWSConfig holds credentials, region, endpoint and style config ---
class AWSConfig {
region;
accessKeyId;
secretAccessKey;
sessionToken;
endpoint;
forcePathStyle;
static fromEnvironment(e) {
let r = __ENV.AWS_REGION,
s = __ENV.AWS_ACCESS_KEY_ID,
t = __ENV.AWS_SECRET_ACCESS_KEY,
n = __ENV.AWS_SESSION_TOKEN,
o = e?.endpoint,
f = e?.forcePathStyle;
return new AWSConfig({ region: r, accessKeyId: s, secretAccessKey: t, sessionToken: n, endpoint: o, forcePathStyle: f });
}
constructor(e) {
if (!e.region || e.region === "") throw new Error(`invalid AWS region; reason: expected a valid AWS region name (e.g. "us-east-1"), got \`${e.region}\``);
if (!e.accessKeyId || e.accessKeyId === "") throw new Error(`invalid AWS access key ID; reason: expected a non empty string, got \`${e.accessKeyId}\``);
if (e.accessKeyId.length < 16 || e.accessKeyId.length > 128 && e.sessionToken != null) throw new Error(`invalid AWS access key ID; reason: size should be between 16 and 128 characters, got ${e.accessKeyId.length}`);
if (!e.secretAccessKey || e.secretAccessKey === "") throw new Error(`invalid AWS secret access key; reason: expected a non empty string, got \`${e.secretAccessKey}\``);
this.region = e.region;
this.accessKeyId = e.accessKeyId;
this.secretAccessKey = e.secretAccessKey;
e.sessionToken !== void 0 && (this.sessionToken = e.sessionToken);
e.endpoint !== void 0 && (typeof e.endpoint == "string" ? this.endpoint = new Endpoint(e.endpoint) : this.endpoint = e.endpoint);
this.forcePathStyle = !!e.forcePathStyle; // default: false
}
}
// --- Error classes for various network and AWS errors ---
class AWSError extends Error {
code;
constructor(e, r) { super(e), this.name = "AWSError", this.code = r; }
static parseXML(e) {
let r = Q(e);
return new AWSError(r.find("Message").text(), r.find("Code").text());
}
static parse(e) {
let headers = (e && e.headers) ? e.headers : {};
if (headers["Content-Type"] === "application/json") {
let r = e.json() || {},
s = r.Message || r.message || r.__type || "An error occurred on the server side",
t = headers["X-Amzn-Errortype"] || r.__type;
return new AWSError(s, t);
} else return AWSError.parseXML(e.body);
}
}
class DNSError extends Error { code; name; constructor(e, r) { super("DNS Error"), this.name = "DNSError", this.code = r; } }
class TCPError extends Error { code; name; constructor(e, r) { super("TCP Error"), this.name = "TCPError", this.code = r; } }
class TLSError extends Error { code; name; constructor(e, r) { super("TLS Error"), this.name = "TLSError", this.code = r; } }
class HTTP2Error extends Error { code; name; constructor(e, r) { super("HTTP2 Error"), this.name = "HTTP2Error", this.code = r; } }
class InvalidSignatureError extends AWSError { constructor(e, r) { super(e, r), this.name = "InvalidSignatureError"; } }
class S3ServiceError extends AWSError { operation; constructor(e, r, s) { super(e, r), this.name = "S3ServiceError", this.operation = s; } }
// --- S3Client: main class for S3 operations ---
class S3Client {
awsConfig;
serviceName;
baseRequestParams = { responseType: "text" };
_endpoint;
signature;
forcePathStyle;
constructor(e) {
this.awsConfig = e;
this.serviceName = "s3";
e.endpoint != null && (this._endpoint = e.endpoint);
this.forcePathStyle = !!e.forcePathStyle;
this.signature = new SignatureV4({
service: this.serviceName,
region: this.awsConfig.region,
credentials: {
accessKeyId: this.awsConfig.accessKeyId,
secretAccessKey: this.awsConfig.secretAccessKey,
sessionToken: this.awsConfig.sessionToken
},
uriEscapePath: false,
applyChecksum: true
});
}
get endpoint() {
// Default endpoint if not set
if (this._endpoint == null) {
this._endpoint = new Endpoint(`https://${this.serviceName}.${this.awsConfig.region}.amazonaws.com`);
}
return this._endpoint;
}
set endpoint(e) { this._endpoint = e; }
// --- Helper: Build URL and host for path-style or virtual-hosted-style ---
_buildRequestParams(bucket, key = "", extra = {}) {
// Returns { endpoint, path }
let endpoint, path;
if (this.forcePathStyle) {
// Path-style: https://endpoint/bucket/key
endpoint = this.endpoint;
path = `/${bucket}${key ? "/" + encodeURIComponent(key) : "/"}`;
} else {
// Virtual-hosted-style: https://bucket.endpoint/key
endpoint = this.endpoint.copy();
endpoint.hostname = `${bucket}.${this.endpoint.hostname}`;
path = key ? `/${encodeURIComponent(key)}` : "/";
}
return { endpoint, path, ...extra };
}
// --- List all buckets (only works with AWS S3 or compatible) ---
async listBuckets() {
let r = this.signature.sign({ method: "GET", endpoint: this.endpoint, path: "/", headers: {} }, {});
let s = await p.asyncRequest("GET", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(s, "ListBuckets");
let t = [];
Q(s.body).find("Buckets").children().each((o, a) => {
let c = {};
a.children().forEach(d => {
switch (d.nodeName().toLowerCase()) {
case "name": c.name = d.textContent(); break;
case "creationdate": c.creationDate = Date.parse(d.textContent()); break;
}
});
t.push(c);
});
return t;
}
// --- List objects in a bucket, with full pagination support ---
async listObjects(bucket, prefix) {
let allObjects = [];
let continuationToken = undefined;
do {
let query = { "list-type": "2", prefix: prefix || "" };
if (continuationToken) query["continuation-token"] = continuationToken;
let { endpoint, path } = this._buildRequestParams(bucket, "", {});
let r = this.signature.sign({
method: "GET",
endpoint,
path,
query,
headers: {}
}, {});
let s = await p.asyncRequest("GET", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(s, "ListObjectsV2");
let doc = Q(s.body);
doc.find("Contents").each((a, c) => {
let d = {};
c.children().forEach(h => {
switch (h.nodeName().toLowerCase()) {
case "key": d.key = h.textContent(); break;
case "lastmodified": d.lastModified = Date.parse(h.textContent()); break;
case "etag": d.etag = h.textContent(); break;
case "size": d.size = parseInt(h.textContent()); break;
case "storageclass": d.storageClass = h.textContent(); break;
// Add more fields if needed
}
});
allObjects.push(d);
});
// Pagination: NextContinuationToken
let nextTokenElem = doc.find("NextContinuationToken");
continuationToken = nextTokenElem.text() || undefined;
} while (continuationToken);
return allObjects;
}
// --- Download an object and parse all relevant metadata ---
async getObject(bucket, key, s = {}) {
let { endpoint, path } = this._buildRequestParams(bucket, key, {});
let r = this.signature.sign({
method: "GET",
endpoint,
path,
headers: { ...s }
}, {});
let o = "text";
"Accept" in s && s.Accept !== void 0 && s.Accept === "application/octet-stream" && (o = "binary");
let a = await p.asyncRequest("GET", r.url, null, { ...this.baseRequestParams, headers: r.headers, responseType: o });
this.handleError(a, "GetObject");
let headers = (a && a.headers) ? a.headers : {};
let etag = getHeader(headers, "ETag");
return new S3Object(
key,
Date.parse(headers["Last-Modified"]),
etag,
parseInt(headers["Content-Length"]),
headers["X-Amz-Storage-Class"] ?? "STANDARD",
a ? a.body : null,
{
contentType: headers["Content-Type"],
contentEncoding: headers["Content-Encoding"],
contentDisposition: headers["Content-Disposition"],
cacheControl: headers["Cache-Control"],
expires: headers["Expires"],
versionId: headers["x-amz-version-id"],
serverSideEncryption: headers["x-amz-server-side-encryption"],
websiteRedirectLocation: headers["x-amz-website-redirect-location"],
replicationStatus: headers["x-amz-replication-status"],
objectLockMode: headers["x-amz-object-lock-mode"],
objectLockRetainUntilDate: headers["x-amz-object-lock-retain-until-date"],
objectLockLegalHoldStatus: headers["x-amz-object-lock-legal-hold"],
customMeta: Object.fromEntries(Object.entries(headers).filter(([k]) => k.toLowerCase().startsWith("x-amz-meta-"))),
// Add more fields as needed
}
);
}
// --- Upload an object (PUT) with optional metadata ---
async putObject(bucket, key, body, t) {
let { endpoint, path } = this._buildRequestParams(bucket, key, {});
let r = this.signature.sign({
method: "PUT",
endpoint,
path,
headers: {
Host: endpoint.host,
...t?.contentDisposition && { "Content-Disposition": t.contentDisposition },
...t?.contentEncoding && { "Content-Encoding": t.contentEncoding },
...t?.contentLength && { "Content-Length": t.contentLength },
...t?.contentMD5 && { "Content-MD5": t.contentMD5 },
...t?.contentType && { "Content-Type": t.contentType },
...t?.cacheControl && { "Cache-Control": t.cacheControl },
...t?.expires && { "Expires": t.expires },
...t?.storageClass && { "x-amz-storage-class": t.storageClass },
...t?.serverSideEncryption && { "x-amz-server-side-encryption": t.serverSideEncryption },
...t?.websiteRedirectLocation && { "x-amz-website-redirect-location": t.websiteRedirectLocation },
...t?.replicationStatus && { "x-amz-replication-status": t.replicationStatus },
...t?.objectLockMode && { "x-amz-object-lock-mode": t.objectLockMode },
...t?.objectLockRetainUntilDate && { "x-amz-object-lock-retain-until-date": t.objectLockRetainUntilDate },
...t?.objectLockLegalHoldStatus && { "x-amz-object-lock-legal-hold": t.objectLockLegalHoldStatus },
...t?.customMeta && Object.fromEntries(Object.entries(t.customMeta).map(([k, v]) => [`x-amz-meta-${k}`, v])),
},
body: body
}, {});
let a = await p.asyncRequest("PUT", r.url, r.body, { ...this.baseRequestParams, headers: r.headers });
this.handleError(a, "PutObject");
let headers = (a && a.headers) ? a.headers : {};
let t_etag = getHeader(headers, "ETag");
return S3UploadedObject.fromResponse(a, key, t_etag);
}
// --- Delete an object ---
async deleteObject(bucket, key) {
let { endpoint, path } = this._buildRequestParams(bucket, key, {});
let r = this.signature.sign({
method: "DELETE",
endpoint,
path,
headers: {}
}, {});
let a = await p.asyncRequest("DELETE", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(a, "DeleteObject");
}
// --- Copy an object (server-side) ---
async copyObject(srcBucket, srcKey, destBucket, destKey) {
let { endpoint, path } = this._buildRequestParams(destBucket, destKey, {});
let r = this.signature.sign({
method: "PUT",
endpoint,
path,
headers: { "x-amz-copy-source": `/${srcBucket}/${srcKey}` }
}, {});
let a = await p.asyncRequest("PUT", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(a, "CopyObject");
// Optionally parse response for metadata
}
// --- Multipart upload: initiate ---
async createMultipartUpload(bucket, key) {
let { endpoint, path } = this._buildRequestParams(bucket, key, {});
let r = this.signature.sign({
method: "POST",
endpoint,
path,
headers: {},
query: { uploads: "" }
}, {});
let a = await p.asyncRequest("POST", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(a, "CreateMultipartUpload");
return new S3MultipartUpload(key, Q(a.body).find("UploadId").text());
}
// --- Multipart upload: upload part ---
async uploadPart(bucket, key, uploadId, partNumber, body) {
let { endpoint, path } = this._buildRequestParams(bucket, key, {});
let r = this.signature.sign({
method: "PUT",
endpoint,
path,
headers: {},
body: body,
query: { partNumber: `${partNumber}`, uploadId: `${uploadId}` }
}, {});
let a = await p.asyncRequest("PUT", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(a, "UploadPart");
let headers = (a && a.headers) ? a.headers : {};
let size = body.byteLength !== undefined ? body.byteLength : body.length;
let eTag = getHeader(headers, "ETag");
return new S3Part(partNumber, eTag, size);
}
// --- Multipart upload: complete ---
async completeMultipartUpload(bucket, key, uploadId, parts) {
let xml = `<CompleteMultipartUpload>${parts.map(h => `<Part><PartNumber>${h.partNumber}</Part><ETag>${h.eTag}</ETag></Part>`).join("")}</CompleteMultipartUpload>`;
let { endpoint, path } = this._buildRequestParams(bucket, key, {});
let r = this.signature.sign({
method: "POST",
endpoint,
path,
headers: {},
body: xml,
query: { uploadId: `${uploadId}` }
}, {});
let a = await p.asyncRequest("POST", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(a, "CompleteMultipartUpload");
let doc = Q(a.body);
let eTag = doc.find("ETag").text();
let size = parts.reduce((sum, p) => sum + (p.size || 0), 0);
return { eTag, size };
}
// --- Multipart upload: abort ---
async abortMultipartUpload(bucket, key, uploadId) {
let { endpoint, path } = this._buildRequestParams(bucket, key, {});
let r = this.signature.sign({
method: "DELETE",
endpoint,
path,
headers: {},
query: { uploadId: `${uploadId}` }
}, {});
let a = await p.asyncRequest("DELETE", r.url, r.body || null, { ...this.baseRequestParams, headers: r.headers });
this.handleError(a, "AbortMultipartUpload");
}
// --- Error handling for S3 responses ---
handleError(e, r) {
if (!e) throw new S3ServiceError("No response object", "NoResponse", r);
if (e.status >= 200 && e.status < 300 && (!e.error || e.error === "")) return false;
if (e.status == 301 || (e.error && e.error.startsWith("301"))) throw new S3ServiceError("Resource not found", "ResourceNotFound", r);
let n = AWSError.parseXML(e.body);
switch (n.code) {
case "AuthorizationHeaderMalformed": throw new InvalidSignatureError(n.message, n.code);
default: throw new S3ServiceError(n.message, n.code || "unknown", r);
}
}
}
// --- S3 object and multipart helper classes ---
class S3MultipartUpload { key; uploadId; constructor(e, r) { this.key = e; this.uploadId = r; } }
class S3Part {
partNumber;
eTag;
size;
constructor(partNumber, eTag, size) {
this.partNumber = partNumber;
this.eTag = eTag;
this.size = size;
}
}
class S3Object {
key; lastModified; etag; size; storageClass; data; meta;
constructor(key, lastModified, etag, size, storageClass, data, meta = {}) {
this.key = key;
this.lastModified = lastModified;
this.etag = etag;
this.size = size;
this.storageClass = storageClass;
this.data = data;
this.meta = meta; // All additional metadata as an object
}
}
class S3UploadedObject {
key; etag; crc32; crc32c; crc64nvme; sha1; sha256; checksumType; size; meta;
constructor(e, r, s, meta = {}) {
this.key = e;
this.etag = r;
this.crc32 = s?.crc32;
this.crc32c = s?.crc32c;
this.crc64nvme = s?.crc64nvme;
this.sha1 = s?.sha1;
this.sha256 = s?.sha256;
this.checksumType = s?.checksumType || "FULL_OBJECT";
this.size = s?.size;
this.meta = meta;
}
static fromResponse(e, r, t_etag) {
let s = {};
let headers = (e && e.headers) ? e.headers : {};
headers["x-amz-checksum-crc32"] && (s.crc32 = headers["x-amz-checksum-crc32"]);
headers["x-amz-checksum-crc32c"] && (s.crc32c = headers["x-amz-checksum-crc32c"]);
headers["x-amz-checksum-crc64nvme"] && (s.crc64nvme = headers["x-amz-checksum-crc64nvme"]);
headers["x-amz-checksum-sha1"] && (s.sha1 = headers["x-amz-checksum-sha1"]);
headers["x-amz-checksum-sha256"] && (s.sha256 = headers["x-amz-checksum-sha256"]);
headers["x-amz-checksum-algorithm"] && (s.checksumType = headers["x-amz-checksum-algorithm"] === "COMPOSITE" ? "COMPOSITE" : "FULL_OBJECT");
headers["x-amz-object-size"] && (s.size = parseInt(headers["x-amz-object-size"]));
// Parse all standard and custom metadata from headers
let meta = {
contentType: headers["Content-Type"],
contentEncoding: headers["Content-Encoding"],
contentDisposition: headers["Content-Disposition"],
cacheControl: headers["Cache-Control"],
expires: headers["Expires"],
versionId: headers["x-amz-version-id"],
serverSideEncryption: headers["x-amz-server-side-encryption"],
websiteRedirectLocation: headers["x-amz-website-redirect-location"],
replicationStatus: headers["x-amz-replication-status"],
objectLockMode: headers["x-amz-object-lock-mode"],
objectLockRetainUntilDate: headers["x-amz-object-lock-retain-until-date"],
objectLockLegalHoldStatus: headers["x-amz-object-lock-legal-hold"],
customMeta: Object.fromEntries(Object.entries(headers).filter(([k]) => k.toLowerCase().startsWith("x-amz-meta-"))),
};
let t = typeof t_etag !== "undefined"
? t_etag
: getHeader(headers, "ETag");
return new S3UploadedObject(r, t, s, meta);
}
}
// --- AWS Signature V4 implementation (unchanged except for comments) ---
class SignatureV4 {
service; region; credentials; uriEscapePath; applyChecksum;
constructor({ service, region, credentials, uriEscapePath, applyChecksum }) {
this.service = service;
this.region = region;
this.credentials = credentials;
this.uriEscapePath = typeof uriEscapePath == "boolean" ? uriEscapePath : true;
this.applyChecksum = typeof applyChecksum == "boolean" ? applyChecksum : true;
}
sign(e, r = {}) {
let t = { ...{ signingDate: new Date, unsignableHeaders: new Set, signableHeaders: new Set }, ...r },
{ longDate: n, shortDate: o } = getDates(t.signingDate),
a = t.signingService || this.service,
c = t.signingRegion || this.region,
d = `${o}/${c}/${a}/aws4_request`;
e.headers["host"] || (e.headers["host"] = e.endpoint.hostname);
e.headers["x-amz-date"] = n;
this.credentials.sessionToken && (e.headers["x-amz-security-token"] = this.credentials.sessionToken);
ArrayBuffer.isView(e.body) && (e.body = e.body.buffer);
e.body || (e.body = "");
let h = this.computePayloadHash(e);
!("x-amz-content-sha256" in e.headers) && this.applyChecksum && (e.headers["x-amz-content-sha256"] = h);
let y = this.computeCanonicalHeaders(e, t.unsignableHeaders, t.signableHeaders),
A = this.calculateSignature(n, d, this.deriveSigningKey(this.credentials, a, c, o), this.createCanonicalRequest(e, y, h));
e.headers["authorization"] = `AWS4-HMAC-SHA256 Credential=${this.credentials.accessKeyId}/${d}, SignedHeaders=${Object.keys(y).sort().join(";")}, Signature=${A}`;
let g = e.endpoint.href;
e.path && (!g.endsWith("/") && !e.path.startsWith("/") && (g += "/"), g += e.path);
e.query && (g += `?${this.serializeQueryParameters(e.query)}`);
return { url: g, ...e };
}
computeCanonicalHeaders({ headers: e }, r, s) {
let t = {};
for (let n of Object.keys(e).sort()) {
if (e[n] == null) continue;
let o = n.toLowerCase();
if (typeof e[n] == "string") t[o] = e[n].trim().replace(/\s+/g, " ");
}
return t;
}
computePayloadHash({ headers: e, body: r }) {
if (e["x-amz-content-sha256"]) return e["x-amz-content-sha256"];
if (r == null) return "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
if (typeof r == "string" || (typeof ArrayBuffer == "function" && r instanceof ArrayBuffer)) return E.sha256(r, "hex").toLowerCase();
if (ArrayBuffer.isView(r)) return E.sha256(r.buffer, "hex").toLowerCase();
return "UNSIGNED-PAYLOAD";
}
createCanonicalRequest(e, r, s) {
let t = Object.keys(r).sort(),
n = t.map(a => `${a}:${r[a]}`).join("\n"),
o = t.join(";");
return `${e.method}\n${this.computeCanonicalURI(e)}\n${this.computeCanonicalQuerystring(e)}\n${n}\n\n${o}\n${s}`;
}
computeCanonicalURI({ path: e }) {
if (this.uriEscapePath) {
let r = [];
for (let c of e.split("/")) c?.length !== 0 && c !== "." && (c === ".." ? r.pop() : r.push(c));
let s = e?.startsWith("/") ? "/" : "",
t = r.join("/"),
n = r.length > 0 && e?.endsWith("/") ? "/" : "",
o = `${s}${t}${n}`;
return encodeURIComponent(o).replace(/%2F/g, "/");
}
return e;
}
computeCanonicalQuerystring({ query: e = {} }) {
let r = [], s = {};
for (let t of Object.keys(e).sort()) {
r.push(t);
let n = e[t];
typeof n == "string" ? s[t] = `${encodeURIComponent(t)}=${encodeURIComponent(n)}` : Array.isArray(n) && (s[t] = n.slice(0).sort().reduce((o, a) => o.concat([`${encodeURIComponent(t)}=${encodeURIComponent(a)}`]), []).join("&"));
}
return r.map(t => s[t]).filter(t => t).join("&");
}
calculateSignature(e, r, s, t) {
let n = this.createStringToSign(e, r, t);
return E.hmac("sha256", s, n, "hex");
}
createStringToSign(e, r, s) {
let t = E.sha256(s, "hex");
return `AWS4-HMAC-SHA256\n${e}\n${r}\n${t}`;
}
deriveSigningKey(e, r, s, t) {
let n = e.secretAccessKey,
o = E.hmac("sha256", "AWS4" + n, t, "binary"),
a = new Uint8Array(o),
c = E.hmac("sha256", a.buffer, s, "binary"),
d = new Uint8Array(c),
h = E.hmac("sha256", d.buffer, r, "binary"),
y = new Uint8Array(h),
A = E.hmac("sha256", y.buffer, "aws4_request", "binary");
return new Uint8Array(A);
}
serializeQueryParameters(e) {
let s = [], t = {};
for (let n of Object.keys(e).sort()) {
s.push(n);
let o = e[n];
typeof o == "string" ? t[n] = `${encodeURIComponent(n)}=${encodeURIComponent(o)}` : Array.isArray(o) && (t[n] = o.slice(0).sort().reduce((a, c) => a.concat([`${encodeURIComponent(n)}=${encodeURIComponent(c)}`]), []).join("&"));
}
return s.map(n => t[n]).filter(n => n).join("&");
}
}
// --- Helper for AWS date formatting ---
function getDates(date) {
let d = new Date(date);
let longDate = d.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
let shortDate = longDate.slice(0, 8);
return { longDate, shortDate };
}
// --- Export all relevant classes ---
export {
AWSConfig,
S3Client,
S3MultipartUpload,
S3Part,
S3Object,
S3UploadedObject,
AWSError,
DNSError,
TCPError,
TLSError,
HTTP2Error,
InvalidSignatureError,
S3ServiceError
};
import crypto from 'k6/crypto';
import { check } from 'k6';
import { Trend, Counter } from 'k6/metrics';
import { AWSConfig, S3Client } from './s3_lib_path_style.js';
/**
* S3 k6 Test Script
* =================
* This script tests S3-compatible storage by uploading, downloading, and deleting files
* using both multipart and single-part upload methods. It is designed for k6 OSS 1.4.0.
*
* Key features:
* - Configurable file size and multipart part size.
* - Configurable multipart part concurrency.
* - Robust error handling and detailed checks.
* - Per-endpoint testing.
*/
/* =========================
CONFIGURATION PARAMETERS
========================= */
/**
* List of S3 endpoints to test.
* Each entry should specify:
* - label: a short name for the endpoint (used in metrics and file keys)
* - endpoint: the S3 endpoint URL or host:port
* - accessKey: S3 access key
* - secretKey: S3 secret key
* - bucket: the bucket to use for testing
*/
const S3_ENDPOINTS = [
{
label: 'ep1',
endpoint: 's3.sub.domain.de:443',
accessKey: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
secretKey: 'yyyyyyyyyyyyyyyyyyyyyyyyyy',
bucket: 's3-bucket-name',
},
// Add more endpoints as needed
];
/**
* Size of the test file to upload (in megabytes).
* This file will be generated once in setup() and used for all uploads.
*/
const FILE_SIZE_MB = 35; // e.g. 35MB file
/**
* Size of each part in a multipart upload (in megabytes).
* S3 requires parts to be at least 5MB except the last part.
*/
const MULTIPART_PART_SIZE_MB = 10; // e.g. 10MB per part
/**
* Number of parts to upload in parallel during multipart upload.
* This controls concurrency within a single multipart upload.
*/
const MULTIPART_PARALLEL_PARTS = 2;
/* =========================
METRICS
========================= */
export const custom_upload_duration_ms = new Trend('custom_upload_duration_ms', false);
export const custom_upload_simple_duration_ms = new Trend('custom_upload_simple_duration_ms', false);
export const custom_download_duration_ms = new Trend('custom_download_duration_ms', false);
export const custom_delete_duration_ms = new Trend('custom_delete_duration_ms', false);
export const custom_exception = new Counter('custom_exception');
/* =========================
K6 OPTIONS
========================= */
export const options = {
scenarios: {
s3_test: {
executor: 'per-vu-iterations',
vus: 1,
iterations: 2,
maxDuration: '1m30s',
gracefulStop: '5s',
},
},
noConnectionReuse: true,
noVUConnectionReuse: true,
systemTags: [
'proto',
'subproto',
'status',
'method',
// 'url' is intentionally excluded for low cardinality
'group',
'check',
'error',
'error_code',
'tls_version',
'scenario',
'service',
'expected_response',
'testid', // custom from cli run command
'instance', // custom from cli run command
'ip', // enabled instead of name and url because of low cardinality per test
],
};
/* =========================
UTILITY FUNCTIONS
========================= */
/**
* Returns a unique S3 object key for each upload.
* Includes endpoint label, VU, iteration, and a suffix for multipart/single.
*/
function uniqueFileKey(endpointLabel, suffix) {
return `k6_testfile_${endpointLabel}_vu${__VU}_it${__ITER}_${suffix}_${Date.now()}_${Math.floor(Math.random()*1e9)}.bin`;
}
/**
* Helper to log only metadata (not file content) for debugging.
*/
function logMeta(label, obj) {
if (obj && typeof obj === 'object') {
const { data, ...meta } = obj;
console.log(`${label}:`, JSON.stringify(meta));
} else {
console.log(`${label}:`, JSON.stringify(obj));
}
}
/**
* Helper to wrap async S3 calls and catch errors.
* Increments a custom exception counter and logs errors.
*/
async function safeCall(fn, metricTags, expectedErrorCode) {
try {
return await fn();
} catch (e) {
const isExpected = expectedErrorCode && e && e.code === expectedErrorCode;
if (!isExpected) {
custom_exception.add(1, metricTags);
console.error(`[ERROR] ${metricTags.upload_type} (${metricTags.endpoint}):`, e && e.message ? e.message : e);
if (e && e.stack) console.error(e.stack);
}
return { __error: e };
}
}
/* =========================
S3 TEST LOGIC
========================= */
/**
* Multipart upload test:
* - Splits the file into parts of MULTIPART_PART_SIZE_MB.
* - Uploads parts in parallel batches.
* - Completes the multipart upload.
* - Downloads and checks the file.
* - Deletes the file and checks it is gone.
*/
async function s3TestMultipart(s3, bucket, endpointLabel, endpointName, fileBuffer) {
const testFileKey = uniqueFileKey(endpointLabel, "multipart");
const metricTags = { upload_type: "multipart", endpoint: endpointName };
// 1. Multipart upload (parts in parallel)
let uploadResult = await safeCall(async () => {
const partSize = MULTIPART_PART_SIZE_MB * 1024 * 1024;
const uploadStart = Date.now();
const multipartUpload = await s3.createMultipartUpload(bucket, testFileKey);
if (!multipartUpload || !multipartUpload.uploadId) throw new Error("Failed to initiate multipart upload");
// Prepare all parts
let partPromises = [];
let partNumber = 1;
for (let offset = 0; offset < fileBuffer.length; offset += partSize, partNumber++) {
const partData = fileBuffer.slice(offset, Math.min(offset + partSize, fileBuffer.length));
partPromises.push({
partNumber,
promise: s3.uploadPart(bucket, testFileKey, multipartUpload.uploadId, partNumber, partData)
});
}
// Upload parts in parallel batches
let completedParts = [];
for (let i = 0; i < partPromises.length; i += MULTIPART_PARALLEL_PARTS) {
const batch = partPromises.slice(i, i + MULTIPART_PARALLEL_PARTS);
const results = await Promise.all(batch.map(p => p.promise));
for (let j = 0; j < results.length; j++) {
completedParts.push({
partNumber: batch[j].partNumber,
eTag: results[j].eTag,
});
}
}
// Complete upload
const result = await s3.completeMultipartUpload(
bucket,
testFileKey,
multipartUpload.uploadId,
completedParts
);
const uploadEnd = Date.now();
custom_upload_duration_ms.add(uploadEnd - uploadStart, metricTags);
return result;
}, metricTags);
check(uploadResult, {
'multipart upload completed': (r) => !!r && !!r.eTag,
});
// 2. Existence check (getObject)
let object = await safeCall(async () => {
if (!uploadResult || uploadResult.__error) return null;
const downloadStart = Date.now();
const obj = await s3.getObject(bucket, testFileKey);
const downloadEnd = Date.now();
custom_download_duration_ms.add(downloadEnd - downloadStart, metricTags);
return obj;
}, metricTags);
check(object, {
'multipart downloaded object exists': (o) => !!o && !!o.data,
'multipart downloaded object size': (o) => !!o && o.size === fileBuffer.length,
});
// 3. Delete
let delRes = await safeCall(async () => {
if (!object || object.__error) return null;
const deleteStart = Date.now();
const res = await s3.deleteObject(bucket, testFileKey);
const deleteEnd = Date.now();
custom_delete_duration_ms.add(deleteEnd - deleteStart, metricTags);
return res;
}, metricTags);
check(delRes, {
'multipart delete operation completed': (r) => r === undefined,
});
// 4. Non-existence check (getObject, expect error code 'NoSuchKey')
let afterDelete = await safeCall(
async () => {
if (!delRes || delRes.__error) {
return { __error: new Error('Delete failed or not attempted') };
}
return await s3.getObject(bucket, testFileKey);
},
metricTags,
"NoSuchKey"
);
check(afterDelete, {
'multipart file not found after delete': (r) => r && r.__error,
});
}
/**
* Single-part upload test:
* - Uploads the file in a single PUT request.
* - Downloads and checks the file.
* - Deletes the file and checks it is gone.
*/
async function s3TestSimplePut(s3, bucket, endpointLabel, endpointName, fileBuffer) {
const testFileKey = uniqueFileKey(endpointLabel, "simple");
const metricTags = { upload_type: "single", endpoint: endpointName };
// 1. Simple PUT
let uploadResult = await safeCall(async () => {
const uploadStart = Date.now();
const result = await s3.putObject(
bucket,
testFileKey,
fileBuffer,
{ contentType: 'application/octet-stream', contentLength: fileBuffer.length }
);
const uploadEnd = Date.now();
custom_upload_simple_duration_ms.add(uploadEnd - uploadStart, metricTags);
return result;
}, metricTags);
check(uploadResult, {
'simple upload completed': (r) => !!r && !!r.etag,
});
// 2. Existence check (getObject)
let object = await safeCall(async () => {
if (!uploadResult || uploadResult.__error) return null;
const downloadStart = Date.now();
const obj = await s3.getObject(bucket, testFileKey);
const downloadEnd = Date.now();
custom_download_duration_ms.add(downloadEnd - downloadStart, metricTags);
return obj;
}, metricTags);
check(object, {
'simple downloaded object exists': (o) => !!o && !!o.data,
'simple downloaded object size': (o) => !!o && o.size === fileBuffer.length,
});
// 3. Delete
let delRes = await safeCall(async () => {
if (!object || object.__error) return null;
const deleteStart = Date.now();
const res = await s3.deleteObject(bucket, testFileKey);
const deleteEnd = Date.now();
custom_delete_duration_ms.add(deleteEnd - deleteStart, metricTags);
return res;
}, metricTags);
check(delRes, {
'simple delete operation completed': (r) => r === undefined,
});
// 4. Non-existence check (getObject, expect error code 'NoSuchKey')
let afterDelete = await safeCall(
async () => {
if (!delRes || delRes.__error) {
return { __error: new Error('Delete failed or not attempted') };
}
return await s3.getObject(bucket, testFileKey);
},
metricTags,
"NoSuchKey"
);
check(afterDelete, {
'simple file not found after delete': (r) => r && r.__error,
});
}
/* =========================
K6 SETUP AND MAIN LOGIC
========================= */
/**
* k6 setup() function.
* Generates a random file buffer of the configured size (FILE_SIZE_MB).
* This buffer is reused for all uploads in the test.
*/
export function setup() {
const fileSizeBytes = FILE_SIZE_MB * 1024 * 1024;
const bigFileBuffer = new Uint8Array(crypto.randomBytes(fileSizeBytes));
return { bigFileBuffer };
}
/**
* k6 default function.
* For each endpoint, runs:
* - Multipart upload test
* - Single-part upload test
* Sequentially, per VU/iteration.
*/
export default async function (setupData) {
const fileBuffer = setupData.bigFileBuffer;
for (const ep of S3_ENDPOINTS) {
const s3 = new S3Client(new AWSConfig({
region: 'us-east-1',
accessKeyId: ep.accessKey,
secretAccessKey: ep.secretKey,
endpoint: ep.endpoint,
forcePathStyle: true,
}));
await s3TestMultipart(s3, ep.bucket, ep.label, ep.endpoint, fileBuffer);
await s3TestSimplePut(s3, ep.bucket, ep.label, ep.endpoint, fileBuffer);
}
}
Hello,
in my local datacenter we use S3 compatible storage. I wanted to create a k6 script which
However for this I needed "forcePathStyle: true" and this was not possible with the library.
Unfortunately I do not know anything about coding, programming and scripts. Fortunately I had AI access and so I created my k6 script with AI and I asked the AI to modify the library to work with my environment.
I want to share these results - maybe someone with knowledge can create PRs and add the changes to the library or at least use the modified library if needed.
Libraray:
And here is my corresponding k6 script:
At least here are the changes the AI did, listed by the AI itself:
General Structure and Organization
Original: All code is minified/obfuscated, with little whitespace, no comments, and all classes/functions in a single file.
Modified: Code is fully reformatted, with whitespace, indentation, comments, and logical separation of classes and helpers. Imports are grouped at the top.
Imports
Original: Uses import{parseHTML as Q}from"k6/html";import p from"k6/http";import E from"k6/crypto"; (sometimes also parseHTML as Ee).
Modified: Same imports, but only parseHTML as Q is used (no Ee), and grouped at the top.
Endpoint Handling
Original: Class R for endpoint parsing.
Modified: Renamed to Endpoint (class name), with identical logic but improved formatting and comments.
AWSConfigOriginal: Class j for AWS config, with region, credentials, and optional endpoint.
Modified: Renamed to AWSConfig, adds support for forcePathStyle (new property and constructor logic), and passes this option from environment and constructor.
Error Classes
Original: Multiple error classes (l, f, m, _, D, H, C, I, S, O) with custom logic and error codes.
Modified:
All error classes are renamed to more descriptive names (AWSError, DNSError, TCPError, TLSError, HTTP2Error, InvalidSignatureError, S3ServiceError).
Error parsing: AWSError.parseXML and AWSError.parse now use only Q (no Ee), and handle missing headers more robustly.
Error handling: All error classes are simplified, and error codes are handled more cleanly.
S3Client Class
Original: Class F (extends M), with methods for all S3 operations.
Modified: Renamed to S3Client, does not extend a base class (all logic is in one class).
Major changes:
Path-style vs. virtual-hosted-style support: New logic via forcePathStyle and _buildRequestParams helper, allowing compatibility with MinIO and other S3-like services.
All S3 methods (listBuckets, listObjects, getObject, putObject, deleteObject, copyObject, multipart upload methods) are rewritten for clarity and extensibility.
Pagination in listObjects: Now supports full pagination using NextContinuationToken (original only fetched one page).
Metadata extraction: getObject and putObject now extract all standard and custom metadata from headers, including x-amz-meta-* fields.
Helper for case-insensitive header extraction: New getHeader function for robust header parsing.
Multipart upload methods: Now return richer objects with part size, eTag, etc.
S3 Object and Upload Classes
Original: Classes X, $, L, v, G for bucket, object, uploaded object, multipart upload, and part.
Modified:
Renamed to S3MultipartUpload, S3Part, S3Object, S3UploadedObject.
S3Object: Now includes a meta field with all extracted metadata.
S3UploadedObject: Now includes a meta field, and fromResponse parses all standard and custom metadata.
S3Part: Now includes part size.
SignatureV4 Implementation
Original: Class U with all AWS Signature V4 logic.
Modified: Renamed to SignatureV4, code is reformatted and commented, but logic is unchanged except for:
Canonical headers: Now only includes string headers, and is simplified.
Query parameter serialization: Now always uses encodeURIComponent for keys and values.
No support for presigned URLs in modified version (original had presign method).
Helper Functions
Original: Several helpers for date formatting, buffer handling, etc.
Modified: Only getDates is kept, and all helpers are commented and simplified.
Exports
Original: Exports all classes with minified names.
Modified: Exports all classes with clear names, matching the new class names.
Other Notable Changes
Case-insensitive header handling: All header lookups are now case-insensitive.
Custom metadata support: All x-amz-meta-* headers are parsed and returned in objects.
Error handling: Now throws more descriptive errors, and checks for missing response objects.
Comments and documentation: The modified version is heavily commented for maintainability.
Removed unused/obscure error codes and mappings: The error code mapping object (Te) is removed.
Removed/Changed Features
No support for presigned URLs: The presign method is not present in the modified version.
No base class for S3Client: The inheritance from M is removed; all logic is in S3Client.
No minification/obfuscation: The code is now readable and maintainable.
Compatibility
MinIO and S3-like services: The addition of forcePathStyle makes the client compatible with MinIO and other S3-compatible endpoints that require path-style requests.