package cloud.lunarsky.store; import com.sun.net.httpserver.HttpServer; import java.net.InetSocketAddress; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Clock; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Base64; import java.util.Comparator; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.Executors; import java.util.zip.CRC32; import java.util.zip.CRC32C; import java.util.zip.Checksum; public final class HttpTest { private static final String ACCESS = "TESTACCESSKEY123"; private static final String SECRET = "test-secret-key-that-is-at-least-32-characters"; private static final String SECONDARY = "SECONDARYKEY1234"; private static final String SECONDARY_SECRET = "secondary-secret-key-that-is-at-least-32-characters"; private static final String REGION = "us-east-1"; private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC); private static HttpRequest signed(String base, String method, String key, byte[] body) { return signedUri(URI.create(base + "/objects/" + SigV4.encode(key, true)), method, body, Map.of()); } static HttpRequest signedUri(URI uri, String method, byte[] body, Map extra) { return signedUriAs(uri, method, body, extra, ACCESS, SECRET); } private static HttpRequest signedUriAs(URI uri, String method, byte[] body, Map extra, String access, String secret) { String host = uri.getAuthority(); String date = DATE.format(Instant.now()); String hash = SigV4.hex(SigV4.hash(body)); TreeMap signed = new TreeMap<>(extra); signed.put("host", host); signed.put("x-amz-content-sha256", hash); signed.put("x-amz-date", date); String names = String.join(";", signed.keySet()); StringBuilder canonical = new StringBuilder(method).append('\n').append(uri.getRawPath()).append('\n') .append(SigV4.canonicalQuery(uri.getRawQuery())).append('\n'); signed.forEach((name, value) -> canonical.append(name).append(':').append(value).append('\n')); canonical.append('\n').append(names).append('\n').append(hash); String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + SigV4.hex(SigV4.hash(canonical.toString().getBytes(StandardCharsets.UTF_8))); String signature = SigV4.hex(SigV4.hmac( SigV4.signingKey(secret, date.substring(0, 8), REGION), toSign)); HttpRequest.Builder request = HttpRequest.newBuilder(uri) .header("x-amz-date", date) .header("x-amz-content-sha256", hash) .header("authorization", "AWS4-HMAC-SHA256 Credential=" + access + "/" + scope + ",SignedHeaders=" + names + ",Signature=" + signature); extra.forEach(request::header); return request.method(method, body.length == 0 ? HttpRequest.BodyPublishers.noBody() : HttpRequest.BodyPublishers.ofByteArray(body)) .build(); } private static URI presignedUri(URI uri, String method, int expires) { String date = DATE.format(Instant.now()); String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; String query = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=" + SigV4.encode(ACCESS + "/" + scope, false) + "&X-Amz-Date=" + date + "&X-Amz-Expires=" + expires + "&X-Amz-SignedHeaders=host"; String canonical = method + "\n" + uri.getRawPath() + "\n" + SigV4.canonicalQuery(query) + "\nhost:" + uri.getAuthority() + "\n\nhost\nUNSIGNED-PAYLOAD"; String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8))); String signature = SigV4.hex(SigV4.hmac(SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign)); return URI.create(uri + "?" + query + "&X-Amz-Signature=" + signature); } private static void testPresigned(HttpClient client, String base) throws Exception { URI object = URI.create(base + "/objects/presigned-test"); byte[] body = "presigned upload".getBytes(StandardCharsets.UTF_8); status(200, client.send(HttpRequest.newBuilder(presignedUri(object, "PUT", 60)) .PUT(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray())); var read = client.send(HttpRequest.newBuilder(presignedUri(object, "GET", 60)).GET().build(), HttpResponse.BodyHandlers.ofByteArray()); status(200, read); if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Presigned object mismatch"); URI tampered = URI.create(presignedUri(object, "GET", 60).toString().replace("presigned-test", "different")); status(403, client.send(HttpRequest.newBuilder(tampered).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(HttpRequest.newBuilder(presignedUri(object, "DELETE", 60)) .DELETE().build(), HttpResponse.BodyHandlers.ofByteArray())); } private static void testAcl(HttpClient client, String base) throws Exception { byte[] body = "private object".getBytes(StandardCharsets.UTF_8); URI object = URI.create(base + "/objects/acl-test"); URI objectAcl = URI.create(object + "?acl"); status(200, client.send(signedUri(object, "PUT", body, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(), SECONDARY, SECRET), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(signedUriAs(URI.create(base + "/_objectstore/capabilities"), "GET", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(HttpRequest.newBuilder(object).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(objectAcl, "PUT", new byte[0], Map.of("x-amz-grant-read", "id=\"" + SECONDARY + "\"")), HttpResponse.BodyHandlers.ofByteArray())); var read = client.send(signedUriAs(object, "GET", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()); status(200, read); if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("ACL read mismatch"); status(403, client.send(signedUriAs(object, "DELETE", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); var acl = client.send(signedUri(objectAcl, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, acl); if (!acl.body().contains(SECONDARY)) throw new AssertionError("Object ACL grant missing"); status(200, client.send(signedUri(URI.create(object + "?acl&x-id=GetObjectAcl"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(objectAcl, "PUT", acl.body().getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray())); URI bucketAcl = URI.create(base + "/objects?acl"); status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0], Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(HttpRequest.newBuilder( URI.create(base + "/objects?list-type=2")).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(HttpRequest.newBuilder( URI.create(base + "/objects?uploads")).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0], Map.of("x-amz-grant-write", "id=\"" + SECONDARY + "\"")), HttpResponse.BodyHandlers.ofByteArray())); URI uploaded = URI.create(base + "/objects/secondary-upload"); status(200, client.send(signedUriAs(uploaded, "PUT", body, Map.of(), SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(signedUriAs(uploaded, "GET", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(uploaded, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(object, "PUT", body, Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(HttpRequest.newBuilder(object).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(HttpRequest.newBuilder(object).DELETE().build(), HttpResponse.BodyHandlers.ofByteArray())); URI historyBucket = URI.create(base + "/acl-history"); status(200, client.send(signedUri(historyBucket, "PUT", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); byte[] versioning = "Enabled" .getBytes(StandardCharsets.UTF_8); status(200, client.send(signedUri(URI.create(historyBucket + "?versioning"), "PUT", versioning, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); URI versioned = URI.create(historyBucket + "/versioned"); var first = client.send(signedUri(versioned, "PUT", body, Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray()); status(200, first); String oldVersion = first.headers().firstValue("x-amz-version-id").orElseThrow(); status(200, client.send(signedUri(versioned, "PUT", body, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(HttpRequest.newBuilder(versioned).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); URI historical = URI.create(versioned + "?versionId=" + oldVersion); status(200, client.send(HttpRequest.newBuilder(historical).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); URI historicalAcl = URI.create(historical + "&acl"); status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0], Map.of("x-amz-grant-write-acp", "id=\"" + SECONDARY + "\"")), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUriAs(historicalAcl, "PUT", new byte[0], Map.of("x-amz-acl", "public-read"), SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(HttpRequest.newBuilder(historical).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0], Map.of("x-amz-acl", "private")), HttpResponse.BodyHandlers.ofByteArray())); status(403, client.send(HttpRequest.newBuilder(historical).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); } private static void testStreaming(HttpClient client, String base) throws Exception { URI object = URI.create(base + "/objects/streaming-test"); byte[] body = "verified streaming payload".getBytes(StandardCharsets.UTF_8); String date = DATE.format(Instant.now()); String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length"; String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" + "host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode + "\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length + "\n\n" + names + "\n" + mode; byte[] signingKey = SigV4.signingKey(SECRET, date.substring(0, 8), REGION); String seed = SigV4.hex(SigV4.hmac(signingKey, "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8))))); String previous = seed; var encoded = new java.io.ByteArrayOutputStream(); for (byte[] chunk : new byte[][]{body, new byte[0]}) { String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" + previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" + SigV4.hex(SigV4.hash(chunk)); previous = SigV4.hex(SigV4.hmac(signingKey, toSign)); encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n") .getBytes(StandardCharsets.US_ASCII)); encoded.write(chunk); encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII)); } HttpRequest.Builder request = HttpRequest.newBuilder(object) .header("content-encoding", "aws-chunked") .header("x-amz-content-sha256", mode) .header("x-amz-date", date) .header("x-amz-decoded-content-length", Integer.toString(body.length)) .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope + ",SignedHeaders=" + names + ",Signature=" + seed); status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(encoded.toByteArray())) .build(), HttpResponse.BodyHandlers.ofByteArray())); var read = client.send(signedUri(object, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, read); if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Streaming upload mismatch"); status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); } private static void testStreamingTrailer(HttpClient client, String base) throws Exception { URI object = URI.create(base + "/objects/streaming-trailer-test"); byte[] body = "verified trailer payload".getBytes(StandardCharsets.UTF_8); String date = DATE.format(Instant.now()); String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; String trailerName = "x-amz-checksum-xxhash3"; String checksum = encodedChecksum("XXHASH3", body); String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;" + "x-amz-decoded-content-length;x-amz-sdk-checksum-algorithm;x-amz-trailer"; String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" + "host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode + "\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length + "\nx-amz-sdk-checksum-algorithm:XXHASH3\nx-amz-trailer:" + trailerName + "\n\n" + names + "\n" + mode; byte[] key = SigV4.signingKey(SECRET, date.substring(0, 8), REGION); String seed = SigV4.hex(SigV4.hmac(key, "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8))))); String previous = seed; var encoded = new java.io.ByteArrayOutputStream(); for (byte[] chunk : new byte[][]{body, new byte[0]}) { String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" + previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" + SigV4.hex(SigV4.hash(chunk)); previous = SigV4.hex(SigV4.hmac(key, toSign)); encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n") .getBytes(StandardCharsets.US_ASCII)); encoded.write(chunk); if (chunk.length != 0) encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII)); } encoded.write((trailerName + ":" + checksum + "\r\n").getBytes(StandardCharsets.US_ASCII)); String trailerToSign = "AWS4-HMAC-SHA256-TRAILER\n" + date + "\n" + scope + "\n" + previous + "\n" + SigV4.hex(SigV4.hash((trailerName + ":" + checksum + "\n") .getBytes(StandardCharsets.UTF_8))); encoded.write(("x-amz-trailer-signature=" + SigV4.hex(SigV4.hmac(key, trailerToSign)) + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII)); byte[] upload = encoded.toByteArray(); HttpRequest.Builder request = HttpRequest.newBuilder(object) .header("content-encoding", "aws-chunked") .header("x-amz-content-sha256", mode) .header("x-amz-date", date) .header("x-amz-decoded-content-length", Integer.toString(body.length)) .header("x-amz-sdk-checksum-algorithm", "XXHASH3") .header("x-amz-trailer", trailerName) .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope + ",SignedHeaders=" + names + ",Signature=" + seed); status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(upload)).build(), HttpResponse.BodyHandlers.ofByteArray())); var head = client.send(signedUri(object, "HEAD", new byte[0], Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding()); status(200, head); if (!checksum.equals(head.headers().firstValue(trailerName).orElse(""))) throw new AssertionError("Signed checksum trailer was not persisted"); byte[] tampered = upload.clone(); tampered[upload.length - 10] ^= 1; status(400, client.send(HttpRequest.newBuilder(object) .header("content-encoding", "aws-chunked") .header("x-amz-content-sha256", mode) .header("x-amz-date", date) .header("x-amz-decoded-content-length", Integer.toString(body.length)) .header("x-amz-sdk-checksum-algorithm", "XXHASH3") .header("x-amz-trailer", trailerName) .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope + ",SignedHeaders=" + names + ",Signature=" + seed) .PUT(HttpRequest.BodyPublishers.ofByteArray(tampered)).build(), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); } private static void testCapabilities(HttpClient client, String base) throws Exception { URI uri = URI.create(base + "/_objectstore/capabilities"); status(403, client.send(HttpRequest.newBuilder(uri).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); var response = client.send(signedUri(uri, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, response); String compact = response.body().replaceAll("\\s+", ""); if (!compact.contains("\"schemaVersion\":1") || !compact.contains("\"service\":\"lunarsky-objectstore\"") || !compact.contains("\"serviceVersion\":\"" + Version.VALUE + "\"") || !compact.contains("\"storageMode\":\"disk\"") || !compact.contains("\"ListParts\"") || !compact.contains("\"maxObjectBytes\":1024") || !compact.contains("\"maxTotalBytes\":4096") || !compact.contains("\"maxParts\":10000")) throw new AssertionError("Unexpected capability manifest: " + response.body()); if (!response.headers().firstValue("content-type").orElse("").startsWith("application/json") || !"no-store".equals(response.headers().firstValue("cache-control").orElse(""))) throw new AssertionError("Capability response headers missing"); status(400, client.send(signedUri(URI.create(uri + "?extra=1"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); } private static void status(int expected, HttpResponse response) { if (response.statusCode() != expected) { throw new AssertionError("Expected HTTP " + expected + ", got " + response.statusCode() + ": " + (response.body() instanceof byte[] bytes ? new String(bytes, StandardCharsets.UTF_8) : response.body())); } } private static void testObjects(HttpClient client, String base) throws Exception { status(200, client.send(HttpRequest.newBuilder(URI.create(base + "/health")).GET().build(), HttpResponse.BodyHandlers.ofByteArray())); String key = "folder/moon-☾.txt"; byte[] body = "independent storage test".getBytes(StandardCharsets.UTF_8); status(403, client.send(HttpRequest.newBuilder(URI.create(base + "/objects/" + key)) .GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signed(base, "PUT", key, body), HttpResponse.BodyHandlers.ofByteArray())); String other = "folder/stars.txt"; status(200, client.send(signedUri(URI.create(base + "/objects/" + other), "PUT", "stars".getBytes(StandardCharsets.UTF_8), Map.of("content-type", "text/plain")), HttpResponse.BodyHandlers.ofByteArray())); var get = client.send(signed(base, "GET", key, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, get); if (!java.util.Arrays.equals(body, get.body())) throw new AssertionError("GET body mismatch"); if (!"application/octet-stream".equals(get.headers().firstValue("content-type").orElse(""))) throw new AssertionError("Unexpected content type"); var typed = client.send(signed(base, "GET", other, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, typed); if (!"text/plain".equals(typed.headers().firstValue("content-type").orElse(""))) throw new AssertionError("Stored content type missing"); var partial = client.send(signedUri(URI.create(base + "/objects/" + other), "GET", new byte[0], Map.of("range", "bytes=1-3")), HttpResponse.BodyHandlers.ofByteArray()); status(206, partial); if (!"tar".equals(new String(partial.body(), StandardCharsets.UTF_8)) || !"bytes 1-3/5".equals(partial.headers().firstValue("content-range").orElse(""))) throw new AssertionError("Range response mismatch"); status(416, client.send(signedUri(URI.create(base + "/objects/" + other), "GET", new byte[0], Map.of("range", "bytes=20-30")), HttpResponse.BodyHandlers.ofByteArray())); } private static void testListing(HttpClient client, String base) throws Exception { var listed = client.send(signedUri(URI.create(base + "/objects?list-type=2&prefix=folder%2F"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (listed.statusCode() != 200 || !listed.body().contains("folder/stars.txt") || !listed.body().contains("folder/moon-☾.txt")) throw new AssertionError("ListObjectsV2 failed: " + listed.body()); var page = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=1"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (page.statusCode() != 200 || !page.body().contains("true")) throw new AssertionError("List pagination failed: " + page.body()); String token = page.body().split("")[1].split("")[0]; var next = client.send(signedUri(URI.create(base + "/objects?list-type=2&continuation-token=" + token), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (next.statusCode() != 200 || !next.body().contains("folder/stars.txt")) throw new AssertionError("List continuation failed: " + next.body()); var grouped = client.send(signedUri(URI.create(base + "/objects?list-type=2&delimiter=%2F"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (grouped.statusCode() != 200 || !grouped.body().contains("folder/") || grouped.body().contains("")) throw new AssertionError("Delimiter listing failed: " + grouped.body()); var encoded = client.send(signedUri(URI.create(base + "/objects?list-type=2&encoding-type=url"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (encoded.statusCode() != 200 || !encoded.body().contains("folder%2Fmoon-%E2%98%BE.txt")) throw new AssertionError("Encoded listing failed: " + encoded.body()); var emptyPage = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=0"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (emptyPage.statusCode() != 200 || !emptyPage.body().contains("0")) throw new AssertionError("Empty list page failed: " + emptyPage.body()); } private static void testCopy(HttpClient client, String base) throws Exception { String source = "folder/copy source.txt"; String target = "folder/copied.txt"; byte[] body = "copy source".getBytes(StandardCharsets.UTF_8); status(200, client.send(signedUri(URI.create(base + "/objects/" + SigV4.encode(source, true)), "PUT", body, Map.of("content-type", "text/plain")), HttpResponse.BodyHandlers.ofByteArray())); String header = "/objects/" + SigV4.encode(source, true); var result = client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", header)), HttpResponse.BodyHandlers.ofString()); if (result.statusCode() != 200 || !result.body().contains("") || !result.body().contains(""")) throw new AssertionError("CopyObject failed: " + result.body()); var copied = client.send(signed(base, "GET", target, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, copied); if (!java.util.Arrays.equals(body, copied.body()) || !"text/plain".equals(copied.headers().firstValue("content-type").orElse(""))) throw new AssertionError("Copied body or content type mismatch"); status(200, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", header, "x-amz-metadata-directive", "REPLACE", "content-type", "text/markdown")), HttpResponse.BodyHandlers.ofByteArray())); var replaced = client.send(signed(base, "GET", target, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, replaced); if (!java.util.Arrays.equals(body, replaced.body()) || !"text/markdown".equals(replaced.headers().firstValue("content-type").orElse(""))) throw new AssertionError("REPLACE metadata directive failed"); status(200, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", "/objects/" + target)), HttpResponse.BodyHandlers.ofByteArray())); status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", "/objects/missing")), HttpResponse.BodyHandlers.ofByteArray())); status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", "/other/source")), HttpResponse.BodyHandlers.ofByteArray())); status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", "/objects/source?versionId=1")), HttpResponse.BodyHandlers.ofByteArray())); status(501, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", header, "x-amz-metadata-directive", "REPLACE", "content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(signed(base, "DELETE", source, new byte[0]), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(signed(base, "DELETE", target, new byte[0]), HttpResponse.BodyHandlers.ofByteArray())); } private static String encodedChecksum(String algorithm, byte[] body) throws Exception { if (algorithm.startsWith("XXHASH")) { var checksum = new XxHashes(algorithm); checksum.update(body, 0, body.length); return Base64.getEncoder().encodeToString(checksum.digest()); } if (algorithm.startsWith("CRC")) { Checksum checksum = switch (algorithm) { case "CRC32" -> new CRC32(); case "CRC32C" -> new CRC32C(); case "CRC64NVME" -> new Crc64Nvme(); default -> throw new IllegalArgumentException(algorithm); }; checksum.update(body, 0, body.length); long value = checksum.getValue(); byte[] bytes = new byte[algorithm.equals("CRC64NVME") ? 8 : 4]; for (int i = bytes.length - 1; i >= 0; i--) { bytes[i] = (byte) value; value >>>= 8; } return Base64.getEncoder().encodeToString(bytes); } String name = algorithm.equals("SHA1") ? "SHA-1" : algorithm.equals("SHA256") ? "SHA-256" : algorithm.equals("SHA512") ? "SHA-512" : "MD5"; return Base64.getEncoder().encodeToString(java.security.MessageDigest.getInstance(name).digest(body)); } private static void testChecksums(HttpClient client, String base) throws Exception { URI uri = URI.create(base + "/objects/checksum-target"); byte[] body = "checksum payload".getBytes(StandardCharsets.UTF_8); String md5 = encodedChecksum("MD5", body); for (String algorithm : new String[]{"CRC32", "CRC32C", "CRC64NVME", "XXHASH64", "XXHASH3", "XXHASH128", "SHA1", "SHA256", "SHA512", "MD5"}) { String header = "x-amz-checksum-" + algorithm.toLowerCase(java.util.Locale.ROOT); String checksum = encodedChecksum(algorithm, body); var stored = client.send(signedUri(uri, "PUT", body, Map.of("content-md5", md5, header, checksum, "x-amz-sdk-checksum-algorithm", algorithm)), HttpResponse.BodyHandlers.ofByteArray()); status(200, stored); if (!checksum.equals(stored.headers().firstValue(header).orElse(""))) throw new AssertionError("Missing checksum response: " + algorithm); var bad = client.send(signedUri(uri, "PUT", body, Map.of(header, Base64.getEncoder().encodeToString(new byte[algorithm.equals("XXHASH128") ? 16 : algorithm.startsWith("XXHASH") || algorithm.equals("CRC64NVME") ? 8 : algorithm.startsWith("CRC") ? 4 : algorithm.equals("SHA1") ? 20 : algorithm.equals("SHA256") ? 32 : algorithm.equals("SHA512") ? 64 : 16]))), HttpResponse.BodyHandlers.ofString()); if (bad.statusCode() != 400 || !bad.body().contains("BadDigest")) throw new AssertionError("Mismatched " + algorithm + " accepted: " + bad.body()); var unchanged = client.send(signedUri(uri, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, unchanged); if (!java.util.Arrays.equals(body, unchanged.body())) throw new AssertionError("Bad checksum replaced stored object"); var head = client.send(signedUri(uri, "HEAD", new byte[0], Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding()); status(200, head); if (!checksum.equals(head.headers().firstValue(header).orElse(""))) throw new AssertionError("Checksum was not persisted: " + algorithm); status(400, client.send(signedUri(uri, "HEAD", new byte[0], Map.of("x-amz-checksum-mode", "INVALID")), HttpResponse.BodyHandlers.discarding())); } var badMd5 = client.send(signedUri(uri, "PUT", body, Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString()); if (badMd5.statusCode() != 400 || !badMd5.body().contains("BadDigest")) throw new AssertionError("Mismatched Content-MD5 accepted"); status(400, client.send(signedUri(uri, "PUT", body, Map.of("content-md5", "invalid")), HttpResponse.BodyHandlers.ofByteArray())); status(400, client.send(signedUri(uri, "PUT", body, Map.of("x-amz-checksum-crc32", encodedChecksum("CRC32", body), "x-amz-sdk-checksum-algorithm", "CRC32C")), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(uri, "PUT", body, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var automatic = client.send(signedUri(uri, "HEAD", new byte[0], Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding()); status(200, automatic); if (!encodedChecksum("CRC64NVME", body).equals(automatic.headers() .firstValue("x-amz-checksum-crc64nvme").orElse(""))) throw new AssertionError("Default CRC64NVME checksum was not persisted"); status(204, client.send(signedUri(uri, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); } private static void testMultipart(HttpClient client, String base) throws Exception { String movie = "folder/video.mp4"; URI initiate = URI.create(base + "/objects/" + movie + "?uploads="); var created = client.send(signedUri(initiate, "POST", new byte[0], Map.of("content-type", "video/mp4")), HttpResponse.BodyHandlers.ofString()); if (created.statusCode() != 200) throw new AssertionError("Multipart initiation failed: " + created.body()); String upload = created.body().split("")[1].split("")[0]; byte[] first = "hello ".getBytes(StandardCharsets.UTF_8); byte[] second = "world".getBytes(StandardCharsets.UTF_8); var partOne = client.send(signedUri(URI.create(base + "/objects/" + movie + "?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie + "?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, partOne); status(200, partTwo); var rejectedPart = client.send(signedUri(URI.create(base + "/objects/" + movie + "?partNumber=1&uploadId=" + upload), "PUT", first, Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString()); if (rejectedPart.statusCode() != 400 || !rejectedPart.body().contains("BadDigest")) throw new AssertionError("Mismatched part Content-MD5 accepted"); var parts = client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload + "&max-parts=1"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (parts.statusCode() != 200 || !parts.body().contains("true") || !parts.body().contains("1")) throw new AssertionError("Multipart part listing failed: " + parts.body()); var uploads = client.send(signedUri(URI.create(base + "/objects?uploads&prefix=folder%2F"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (uploads.statusCode() != 200 || !uploads.body().contains("" + upload + "")) throw new AssertionError("Multipart upload listing failed: " + uploads.body()); String completion = "1" + partOne.headers().firstValue("etag").orElseThrow() + "2" + partTwo.headers().firstValue("etag").orElseThrow() + ""; status(200, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload), "POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var assembled = client.send(signed(base, "GET", movie, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, assembled); if (!"hello world".equals(new String(assembled.body(), StandardCharsets.UTF_8)) || !"video/mp4".equals(assembled.headers().firstValue("content-type").orElse(""))) throw new AssertionError("Completed multipart object mismatch"); status(404, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var abandoned = client.send(signedUri(URI.create(base + "/objects/abandoned?uploads="), "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); String abandonedId = abandoned.body().split("")[1].split("")[0]; status(204, client.send(signedUri(URI.create(base + "/objects/abandoned?uploadId=" + abandonedId), "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); } private static void testDelete(HttpClient client, String base) throws Exception { String key = "folder/moon-☾.txt"; var head = client.send(signed(base, "HEAD", key, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, head); if (head.body().length != 0) throw new AssertionError("HEAD returned a body"); status(204, client.send(signed(base, "DELETE", key, new byte[0]), HttpResponse.BodyHandlers.ofByteArray())); status(404, client.send(signed(base, "GET", key, new byte[0]), HttpResponse.BodyHandlers.ofByteArray())); } private static void testAttributes(HttpClient client, String base) throws Exception { URI object = URI.create(base + "/objects/attributes.txt"); byte[] body = "object attributes".getBytes(StandardCharsets.UTF_8); status(200, client.send(signedUri(object, "PUT", body, Map.of( "x-amz-meta-project", "LunarSky", "x-amz-tagging", "kind=test&phase=one")), HttpResponse.BodyHandlers.ofByteArray())); var get = client.send(signedUri(object, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, get); if (!"LunarSky".equals(get.headers().firstValue("x-amz-meta-project").orElse("")) || !"2".equals(get.headers().firstValue("x-amz-tagging-count").orElse(""))) throw new AssertionError("Stored attributes missing from GET"); var tagging = URI.create(object + "?tagging"); var originalTags = client.send(signedUri(URI.create(tagging + "&x-id=GetObjectTagging"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, originalTags); if (!originalTags.body().contains("kindtest")) throw new AssertionError("Object tags were not stored"); byte[] replacement = "stagetwo" .getBytes(StandardCharsets.UTF_8); status(200, client.send(signedUri(tagging, "PUT", replacement, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(400, client.send(signedUri(tagging, "PUT", replacement, Map.of("content-md5", Base64.getEncoder().encodeToString(new byte[16]))), HttpResponse.BodyHandlers.ofByteArray())); var replaced = client.send(signedUri(tagging, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (!replaced.body().contains("stagetwo") || replaced.body().contains("kind")) throw new AssertionError("Tag replacement failed"); status(400, client.send(signedUri(tagging, "PUT", "]>" .getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(signedUri(tagging, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var cleared = client.send(signedUri(tagging, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (!cleared.body().contains("")) throw new AssertionError("Tag deletion failed"); status(200, client.send(signedUri(URI.create(base + "/objects/attributes-copy.txt"), "PUT", new byte[0], Map.of("x-amz-copy-source", "/objects/attributes.txt")), HttpResponse.BodyHandlers.ofByteArray())); var copied = client.send(signed(base, "GET", "attributes-copy.txt", new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse(""))) throw new AssertionError("Copy lost user metadata"); status(400, client.send(signedUri(object, "PUT", body, Map.of("x-amz-meta-bad", "a".repeat(2100))), HttpResponse.BodyHandlers.ofByteArray())); } private static void testBuckets(HttpClient client, String base) throws Exception { var root = URI.create(base + "/"); var existing = client.send(signedUri(root, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, existing); if (!existing.body().contains("objects")) throw new AssertionError("Default bucket absent from service listing"); var bucket = URI.create(base + "/second-bucket"); status(200, client.send(signedUri(URI.create(bucket + "?x-id=CreateBucket"), "PUT", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(409, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var upload = client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploads"), "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, upload); String uploadId = upload.body().split("")[1].split("")[0]; status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploadId=" + uploadId), "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); byte[] body = "second bucket".getBytes(StandardCharsets.UTF_8); var object = URI.create(base + "/second-bucket/one.txt"); status(200, client.send(signedUri(object, "PUT", body, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var read = client.send(signedUri(object, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, read); if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Second bucket read failed"); var crossCopy = URI.create(base + "/second-bucket/copied.txt"); status(200, client.send(signedUri(crossCopy, "PUT", new byte[0], Map.of("x-amz-copy-source", "/objects/attributes.txt")), HttpResponse.BodyHandlers.ofByteArray())); var copied = client.send(signedUri(crossCopy, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse(""))) throw new AssertionError("Cross-bucket copy lost metadata"); var listed = client.send(signedUri(URI.create(base + "/second-bucket?list-type=2"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (!listed.body().contains("one.txt")) throw new AssertionError("Second bucket listing failed"); status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(signedUri(crossCopy, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(204, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); status(404, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); } private static void testVersioning(HttpClient client, String base) throws Exception { URI bucket = URI.create(base + "/version-bucket"); URI configuration = URI.create(bucket + "?versioning"); URI object = URI.create(bucket + "/note.txt"); status(200, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var initial = client.send(signedUri(configuration, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); if (!initial.body().contains("")) throw new AssertionError("New bucket versioning state"); status(200, client.send(signedUri(object, "PUT", new byte[]{1}, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); byte[] enable = "Enabled" .getBytes(StandardCharsets.UTF_8); status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var first = client.send(signedUri(object, "PUT", new byte[]{2}, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, first); String id = first.headers().firstValue("x-amz-version-id").orElseThrow(); var second = client.send(signedUri(object, "PUT", new byte[]{3}, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, second); if (id.equals(second.headers().firstValue("x-amz-version-id").orElseThrow())) throw new AssertionError("Version IDs repeated"); var old = client.send(signedUri(URI.create(object + "?versionId=" + id), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, old); if (old.body()[0] != 2) throw new AssertionError("Historical object body"); byte[] tagged = "kindold" .getBytes(StandardCharsets.UTF_8); status(200, client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "PUT", tagged, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var oldTags = client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, oldTags); if (!oldTags.body().contains("old")) throw new AssertionError("Versioned tagging failed"); var copied = client.send(signedUri(URI.create(bucket + "/copied.txt"), "PUT", new byte[0], Map.of("x-amz-copy-source", "/version-bucket/note.txt?versionId=" + id)), HttpResponse.BodyHandlers.ofByteArray()); status(200, copied); if (!id.equals(copied.headers().firstValue("x-amz-copy-source-version-id").orElse(""))) throw new AssertionError("Copy did not report source version"); var copyBody = client.send(signedUri(URI.create(bucket + "/copied.txt"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, copyBody); if (copyBody.body()[0] != 2) throw new AssertionError("Copy of old version failed"); var deleted = client.send(signedUri(object, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(204, deleted); String marker = deleted.headers().firstValue("x-amz-version-id").orElseThrow(); var hidden = client.send(signedUri(object, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(404, hidden); if (!"true".equals(hidden.headers().firstValue("x-amz-delete-marker").orElse(""))) throw new AssertionError("Missing delete marker response header"); status(405, client.send(signedUri(URI.create(object + "?versionId=" + marker), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var listed = client.send(signedUri(URI.create(bucket + "?versions&max-keys=2"), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, listed); if (!listed.body().contains("") || !listed.body().contains("true")) throw new AssertionError("Version listing did not include delete marker"); status(204, client.send(signedUri(URI.create(object + "?versionId=" + marker), "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var restored = client.send(signedUri(object, "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, restored); if (restored.body()[0] != 3) throw new AssertionError("Deleting marker did not restore current version"); byte[] suspend = "Suspended" .getBytes(StandardCharsets.UTF_8); status(200, client.send(signedUri(configuration, "PUT", suspend, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var nullVersion = client.send(signedUri(object, "PUT", new byte[]{4}, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, nullVersion); if (!"null".equals(nullVersion.headers().firstValue("x-amz-version-id").orElse(""))) throw new AssertionError("Suspended write did not produce null version"); status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); URI multipartObject = URI.create(bucket + "/multipart.txt"); var initiated = client.send(signedUri(URI.create(multipartObject + "?uploads"), "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); status(200, initiated); String uploadId = initiated.body().split("")[1].split("")[0]; byte[] partBody = "versioned multipart".getBytes(StandardCharsets.UTF_8); var part = client.send(signedUri(URI.create(multipartObject + "?partNumber=1&uploadId=" + uploadId), "PUT", partBody, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, part); String completion = "1" + part.headers().firstValue("etag").orElseThrow() + ""; var completed = client.send(signedUri(URI.create(multipartObject + "?uploadId=" + uploadId), "POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, completed); String multipartVersion = completed.headers().firstValue("x-amz-version-id").orElseThrow(); status(204, client.send(signedUri(multipartObject, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var retainedMultipart = client.send(signedUri(URI.create(multipartObject + "?versionId=" + multipartVersion), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); status(200, retainedMultipart); if (!java.util.Arrays.equals(partBody, retainedMultipart.body())) throw new AssertionError("Completed multipart version was not retained"); } private static int virtualHostPut(int port, String host, String signedHost, byte[] body) throws Exception { URI signed = URI.create("http://" + signedHost + ":" + port + "/photos/cat.jpg"); HttpRequest request = signedUri(signed, "PUT", body, Map.of()); String headers = "PUT /photos/cat.jpg HTTP/1.1\r\nHost: " + host + ":" + port + "\r\nAuthorization: " + request.headers().firstValue("authorization").orElseThrow() + "\r\nx-amz-date: " + request.headers().firstValue("x-amz-date").orElseThrow() + "\r\nx-amz-content-sha256: " + request.headers().firstValue("x-amz-content-sha256").orElseThrow() + "\r\nContent-Length: " + body.length + "\r\nConnection: close\r\n\r\n"; try (var socket = new java.net.Socket("127.0.0.1", port)) { socket.setSoTimeout(5000); socket.getOutputStream().write(headers.getBytes(StandardCharsets.US_ASCII)); socket.getOutputStream().write(body); String response = new String(socket.getInputStream().readAllBytes(), StandardCharsets.ISO_8859_1); return Integer.parseInt(response.split(" ", 3)[1]); } } private static void testGatewayConcurrency(DiskStore store, HttpClient client, java.util.concurrent.Executor executor) throws Exception { HttpServer limited = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16); var app = new Main(store, new MultipartStore(store), new SigV4(ACCESS, SECRET, REGION, Clock.systemUTC()), "objects", ClientLimits.disabled(), 1, ""); limited.setExecutor(executor); limited.createContext("/", app::handle); limited.start(); int port = limited.getAddress().getPort(); String base = "http://127.0.0.1:" + port; byte[] body = {42}; HttpRequest request = signedUri(URI.create(base + "/objects/held"), "PUT", body, Map.of()); try (var socket = new java.net.Socket("127.0.0.1", port)) { socket.setSoTimeout(5000); String headers = "PUT /objects/held HTTP/1.1\r\nHost: 127.0.0.1:" + port + "\r\nAuthorization: " + request.headers().firstValue("authorization").orElseThrow() + "\r\nx-amz-date: " + request.headers().firstValue("x-amz-date").orElseThrow() + "\r\nx-amz-content-sha256: " + request.headers().firstValue("x-amz-content-sha256").orElseThrow() + "\r\nContent-Length: 1\r\nConnection: close\r\n\r\n"; socket.getOutputStream().write(headers.getBytes(StandardCharsets.US_ASCII)); boolean refused = false; for (int attempt = 0; attempt < 50 && !refused; attempt++) { int status = client.send(HttpRequest.newBuilder(URI.create(base + "/health")).GET().build(), HttpResponse.BodyHandlers.discarding()).statusCode(); refused = status == 503; if (!refused) Thread.sleep(10); } if (!refused) throw new AssertionError("Configured gateway concurrency cap was not enforced"); socket.getOutputStream().write(body); String finished = new String(socket.getInputStream().readAllBytes(), StandardCharsets.ISO_8859_1); if (!finished.startsWith("HTTP/1.1 200 ")) throw new AssertionError("Held request did not complete after releasing its body"); } finally { limited.stop(0); } } public static void main(String[] args) throws Exception { Path root = Files.createTempDirectory("store-http-test-"); var executor = Executors.newVirtualThreadPerTaskExecutor(); HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16); DiskStore store = new DiskStore(root, 1024, 4096); try { var app = new Main(store, new MultipartStore(store), new SigV4(Map.of(ACCESS, SECRET, SECONDARY, SECONDARY_SECRET), ACCESS, REGION, Clock.systemUTC()), "objects", ClientLimits.disabled(), 16, "s3.test"); server.setExecutor(executor); server.createContext("/", app::handle); server.start(); String base = "http://127.0.0.1:" + server.getAddress().getPort(); HttpClient client = HttpClient.newHttpClient(); testCapabilities(client, base); testPresigned(client, base); testStreaming(client, base); testStreamingTrailer(client, base); testObjects(client, base); testListing(client, base); testCopy(client, base); testChecksums(client, base); testMultipart(client, base); testAttributes(client, base); testDelete(client, base); testBuckets(client, base); testVersioning(client, base); testAcl(client, base); byte[] hostedBody = "virtual host".getBytes(StandardCharsets.UTF_8); int port = server.getAddress().getPort(); if (virtualHostPut(port, "objects.s3.test", "objects.s3.test", hostedBody) != 200) throw new AssertionError("Signed virtual-hosted upload failed"); try (var opened = store.open("objects", "photos/cat.jpg")) { if (!java.util.Arrays.equals(hostedBody, opened.stream().readAllBytes())) throw new AssertionError("Virtual-hosted bucket or key was parsed incorrectly"); } if (virtualHostPut(port, "objects.s3.test", "other.s3.test", hostedBody) != 403) throw new AssertionError("Changing a signed virtual hostname was accepted"); testGatewayConcurrency(store, client, executor); System.out.println("HTTP tests passed: capabilities, objects, copy, checksums, listing, multipart, attributes, buckets, versioning, ACLs"); } finally { server.stop(0); executor.close(); store.close(); try (var paths = Files.walk(root)) { for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path); } } } }