admin/ObjectStore
1
1 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Release ObjectStore 0.0.8

Browse Source
This commit is contained in:
admin committed 2026-10-10 15:22:21 +02:00
1 parent ed7712a8af
commit d6427fbac9
61 files changed
+7082 -267
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/cloud/lunarsky/store/CliTest.java
+15
View File
@@ -44,6 +44,21 @@ public final class CliTest {
int status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), new PrintStream(output));
if (status != 1 || !output.toString().contains("Object checksum mismatch"))
throw new AssertionError("CLI missed corrupted payload");
Path versionRoot = root.resolve("versioned");
try (var store = new DiskStore(versionRoot, 100, 1000)) {
store.createBucket("versioned-bucket");
store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.ENABLED);
for (byte[] body : new byte[][]{"first".getBytes(StandardCharsets.UTF_8),
"second".getBytes(StandardCharsets.UTF_8)}) {
store.put("versioned-bucket", "example", new ByteArrayInputStream(body), body.length,
SigV4.hex(SigV4.hash(body)), null, false, "text/plain");
}
output.reset();
status = Cli.run(new String[]{"verify"}, versionRoot,
new PrintStream(output), new PrintStream(output));
if (status != 0 || !output.toString().contains("verified_objects=2"))
throw new AssertionError("CLI did not inspect retained versions");
}
System.out.println("CLI tests passed: version, live status, verification, corruption exit code");
} finally {
try (var paths = Files.walk(root)) {
test/cloud/lunarsky/store/ClientLimitsTest.java
+98
View File
@@ -0,0 +1,98 @@
package cloud.lunarsky.store;
import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Clock;
import java.util.Comparator;
import java.util.Map;
import java.util.concurrent.Executors;
public final class ClientLimitsTest {
private static final String ACCESS = "TESTACCESSKEY123";
private static final String SECRET = "test-secret-key-that-is-at-least-32-characters";
private static HttpResponse<String> get(HttpClient client, String base, String path, String ip) throws Exception {
var request = HttpRequest.newBuilder(URI.create(base + path));
if (ip != null) request.header("X-Real-IP", ip);
return client.send(request.GET().build(), HttpResponse.BodyHandlers.ofString());
}
private static void status(int wanted, HttpResponse<?> response) {
if (response.statusCode() != wanted)
throw new AssertionError("Expected " + wanted + ", got " + response.statusCode() + ": " + response.body());
}
private static void exercise(Map<String, String> configuration, boolean trusted) throws Exception {
Path root = Files.createTempDirectory("client-limits-test-");
var executor = Executors.newVirtualThreadPerTaskExecutor();
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16);
DiskStore store = new DiskStore(root, 1024, 4096);
try {
var app = new Main(store, new MultipartStore(store),
new SigV4(Map.of(ACCESS, SECRET), ACCESS, "us-east-1", Clock.systemUTC()),
"objects", ClientLimits.fromEnvironment(configuration));
server.setExecutor(executor);
server.createContext("/", app::handle);
server.start();
String base = "http://127.0.0.1:" + server.getAddress().getPort();
HttpClient client = HttpClient.newHttpClient();
if (trusted) {
status(200, get(client, base, "/health", null));
status(400, get(client, base, "/ready", null));
status(400, get(client, base, "/objects/example", null));
status(400, get(client, base, "/health", "not-an-ip"));
status(400, client.send(HttpRequest.newBuilder(URI.create(base + "/health"))
.header("X-Real-IP", "192.0.2.1")
.header("X-Real-IP", "192.0.2.2")
.GET().build(), HttpResponse.BodyHandlers.ofString()));
status(200, get(client, base, "/health", "192.0.2.1"));
var limited = get(client, base, "/health", "192.0.2.1");
status(503, limited);
if (!"1".equals(limited.headers().firstValue("Retry-After").orElse(null)))
throw new AssertionError("SlowDown response lacks Retry-After");
status(200, get(client, base, "/health", "192.0.2.2"));
} else if (configuration.containsKey("PUBLIC_BYTES_PER_SECOND")) {
long start = System.nanoTime();
byte[] upload = new byte[128];
status(200, client.send(HttpTest.signedUri(URI.create(base + "/objects/bandwidth"),
"PUT", upload, Map.of()), HttpResponse.BodyHandlers.ofString()));
if (System.nanoTime() - start < 800_000_000L)
throw new AssertionError("Upload bytes were not paced");
start = System.nanoTime();
status(200, get(client, base, "/health", "192.0.2.1"));
status(200, get(client, base, "/health", "192.0.2.1"));
if (System.nanoTime() - start < 250_000_000L)
throw new AssertionError("Responses were not paced by the shared byte budget");
} else {
status(200, get(client, base, "/health", "192.0.2.1"));
status(503, get(client, base, "/health", "192.0.2.2"));
}
} finally {
server.stop(0);
executor.close();
store.close();
try (var paths = Files.walk(root)) {
for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path);
}
}
}
public static void main(String[] args) throws Exception {
var disabled = ClientLimits.fromEnvironment(Map.of());
if (disabled == null) throw new AssertionError("Disabled configuration missing");
try { ClientLimits.fromEnvironment(Map.of("PUBLIC_TRUSTED_PROXY_IPS", "127.0.0.1"));
throw new AssertionError("Proxy trust accepted without limits");
} catch (IllegalArgumentException expected) { }
exercise(Map.of("PUBLIC_REQUESTS_PER_SECOND", "1", "PUBLIC_REQUEST_BURST", "1",
"PUBLIC_TRUSTED_PROXY_IPS", "127.0.0.1"), true);
exercise(Map.of("PUBLIC_REQUESTS_PER_SECOND", "1", "PUBLIC_REQUEST_BURST", "1"), false);
exercise(Map.of("PUBLIC_BYTES_PER_SECOND", "64", "PUBLIC_BYTE_BURST", "64"), false);
System.out.println("Client limit tests passed");
}
}
test/cloud/lunarsky/store/ClusterNodeTest.java
+19 -1
View File
@@ -138,7 +138,25 @@ public final class ClusterNodeTest {
throw new AssertionError("Oversized segment response was accepted");
} catch (IOException expected) { }
} finally { oversized.stop(0); }
System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup");
URI stopped = URI.create("http://127.0.0.1:" + oversized.getAddress().getPort());
NodeClient readOnly = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, stopped)), token, null);
try {
readOnly.get(0, id, value.length, SigV4.hash(value));
throw new AssertionError("Read-only access did not detect a stopped node");
} catch (IOException expected) {
require(expected.getMessage().contains("temporarily unreachable"),
"Read-only access did not use the short health probe");
}
NodeClient offline = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, stopped)), token, null);
require(!offline.availableHostsAtLeast(1, false), "Stopped node was reported healthy");
try {
offline.get(0, id, value.length, SigV4.hash(value));
throw new AssertionError("Stopped node was read after a failed health check");
} catch (IOException expected) {
require(expected.getMessage().contains("temporarily unreachable"),
"Read did not skip a recently failed node");
}
System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup, outage fallback");
}
private static void require(boolean condition, String message) {
if (!condition) throw new AssertionError(message);
test/cloud/lunarsky/store/HttpTest.java
+534 -15
View File
@@ -25,6 +25,8 @@ import java.util.zip.Checksum;
public final class HttpTest {
private static final String ACCESS = "TESTACCESSKEY123";
private static final String SECRET = "test-secret-key-that-is-at-least-32-characters";
private static final String SECONDARY = "SECONDARYKEY1234";
private static final String SECONDARY_SECRET = "secondary-secret-key-that-is-at-least-32-characters";
private static final String REGION = "us-east-1";
private static final DateTimeFormatter DATE =
DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);
@@ -33,7 +35,12 @@ public final class HttpTest {
return signedUri(URI.create(base + "/objects/" + SigV4.encode(key, true)), method, body, Map.of());
}
private static HttpRequest signedUri(URI uri, String method, byte[] body, Map<String, String> extra) {
static HttpRequest signedUri(URI uri, String method, byte[] body, Map<String, String> extra) {
return signedUriAs(uri, method, body, extra, ACCESS, SECRET);
}
private static HttpRequest signedUriAs(URI uri, String method, byte[] body, Map<String, String> extra,
String access, String secret) {
String host = uri.getAuthority();
String date = DATE.format(Instant.now());
String hash = SigV4.hex(SigV4.hash(body));
@@ -50,11 +57,11 @@ public final class HttpTest {
String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
+ SigV4.hex(SigV4.hash(canonical.toString().getBytes(StandardCharsets.UTF_8)));
String signature = SigV4.hex(SigV4.hmac(
SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign));
SigV4.signingKey(secret, date.substring(0, 8), REGION), toSign));
HttpRequest.Builder request = HttpRequest.newBuilder(uri)
.header("x-amz-date", date)
.header("x-amz-content-sha256", hash)
.header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/"
.header("authorization", "AWS4-HMAC-SHA256 Credential=" + access + "/"
+ scope + ",SignedHeaders=" + names + ",Signature=" + signature);
extra.forEach(request::header);
return request.method(method, body.length == 0
@@ -63,10 +70,275 @@ public final class HttpTest {
.build();
}
private static void status(int expected, HttpResponse<byte[]> response) {
private static URI presignedUri(URI uri, String method, int expires) {
String date = DATE.format(Instant.now());
String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
String query = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=" +
SigV4.encode(ACCESS + "/" + scope, false) + "&X-Amz-Date=" + date +
"&X-Amz-Expires=" + expires + "&X-Amz-SignedHeaders=host";
String canonical = method + "\n" + uri.getRawPath() + "\n" +
SigV4.canonicalQuery(query) + "\nhost:" + uri.getAuthority() +
"\n\nhost\nUNSIGNED-PAYLOAD";
String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" +
SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)));
String signature = SigV4.hex(SigV4.hmac(SigV4.signingKey(SECRET,
date.substring(0, 8), REGION), toSign));
return URI.create(uri + "?" + query + "&X-Amz-Signature=" + signature);
}
private static void testPresigned(HttpClient client, String base) throws Exception {
URI object = URI.create(base + "/objects/presigned-test");
byte[] body = "presigned upload".getBytes(StandardCharsets.UTF_8);
status(200, client.send(HttpRequest.newBuilder(presignedUri(object, "PUT", 60))
.PUT(HttpRequest.BodyPublishers.ofByteArray(body)).build(),
HttpResponse.BodyHandlers.ofByteArray()));
var read = client.send(HttpRequest.newBuilder(presignedUri(object, "GET", 60)).GET().build(),
HttpResponse.BodyHandlers.ofByteArray());
status(200, read);
if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Presigned object mismatch");
URI tampered = URI.create(presignedUri(object, "GET", 60).toString().replace("presigned-test", "different"));
status(403, client.send(HttpRequest.newBuilder(tampered).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(HttpRequest.newBuilder(presignedUri(object, "DELETE", 60))
.DELETE().build(), HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testAcl(HttpClient client, String base) throws Exception {
byte[] body = "private object".getBytes(StandardCharsets.UTF_8);
URI object = URI.create(base + "/objects/acl-test");
URI objectAcl = URI.create(object + "?acl");
status(200, client.send(signedUri(object, "PUT", body, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(),
SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(),
SECONDARY, SECRET), HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(signedUriAs(URI.create(base + "/_objectstore/capabilities"),
"GET", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET),
HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(HttpRequest.newBuilder(object).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(objectAcl, "PUT", new byte[0],
Map.of("x-amz-grant-read", "id=\"" + SECONDARY + "\"")),
HttpResponse.BodyHandlers.ofByteArray()));
var read = client.send(signedUriAs(object, "GET", new byte[0], Map.of(),
SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray());
status(200, read);
if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("ACL read mismatch");
status(403, client.send(signedUriAs(object, "DELETE", new byte[0], Map.of(),
SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
var acl = client.send(signedUri(objectAcl, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
status(200, acl);
if (!acl.body().contains(SECONDARY)) throw new AssertionError("Object ACL grant missing");
status(200, client.send(signedUri(URI.create(object + "?acl&x-id=GetObjectAcl"),
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(objectAcl, "PUT",
acl.body().getBytes(StandardCharsets.UTF_8), Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
URI bucketAcl = URI.create(base + "/objects?acl");
status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0],
Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(HttpRequest.newBuilder(
URI.create(base + "/objects?list-type=2")).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(HttpRequest.newBuilder(
URI.create(base + "/objects?uploads")).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0],
Map.of("x-amz-grant-write", "id=\"" + SECONDARY + "\"")),
HttpResponse.BodyHandlers.ofByteArray()));
URI uploaded = URI.create(base + "/objects/secondary-upload");
status(200, client.send(signedUriAs(uploaded, "PUT", body, Map.of(),
SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(signedUriAs(uploaded, "GET", new byte[0], Map.of(),
SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(uploaded, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(object, "PUT", body,
Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(HttpRequest.newBuilder(object).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(HttpRequest.newBuilder(object).DELETE().build(),
HttpResponse.BodyHandlers.ofByteArray()));
URI historyBucket = URI.create(base + "/acl-history");
status(200, client.send(signedUri(historyBucket, "PUT", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
byte[] versioning = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"
.getBytes(StandardCharsets.UTF_8);
status(200, client.send(signedUri(URI.create(historyBucket + "?versioning"),
"PUT", versioning, Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
URI versioned = URI.create(historyBucket + "/versioned");
var first = client.send(signedUri(versioned, "PUT", body, Map.of("x-amz-acl", "public-read")),
HttpResponse.BodyHandlers.ofByteArray());
status(200, first);
String oldVersion = first.headers().firstValue("x-amz-version-id").orElseThrow();
status(200, client.send(signedUri(versioned, "PUT", body, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(HttpRequest.newBuilder(versioned).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
URI historical = URI.create(versioned + "?versionId=" + oldVersion);
status(200, client.send(HttpRequest.newBuilder(historical).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
URI historicalAcl = URI.create(historical + "&acl");
status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0],
Map.of("x-amz-grant-write-acp", "id=\"" + SECONDARY + "\"")),
HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUriAs(historicalAcl, "PUT", new byte[0],
Map.of("x-amz-acl", "public-read"), SECONDARY, SECONDARY_SECRET),
HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(HttpRequest.newBuilder(historical).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0],
Map.of("x-amz-acl", "private")), HttpResponse.BodyHandlers.ofByteArray()));
status(403, client.send(HttpRequest.newBuilder(historical).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testStreaming(HttpClient client, String base) throws Exception {
URI object = URI.create(base + "/objects/streaming-test");
byte[] body = "verified streaming payload".getBytes(StandardCharsets.UTF_8);
String date = DATE.format(Instant.now());
String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length";
String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" +
"host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode +
"\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length +
"\n\n" + names + "\n" + mode;
byte[] signingKey = SigV4.signingKey(SECRET, date.substring(0, 8), REGION);
String seed = SigV4.hex(SigV4.hmac(signingKey, "AWS4-HMAC-SHA256\n" + date + "\n" +
scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)))));
String previous = seed;
var encoded = new java.io.ByteArrayOutputStream();
for (byte[] chunk : new byte[][]{body, new byte[0]}) {
String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" +
previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" +
SigV4.hex(SigV4.hash(chunk));
previous = SigV4.hex(SigV4.hmac(signingKey, toSign));
encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n")
.getBytes(StandardCharsets.US_ASCII));
encoded.write(chunk);
encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII));
}
HttpRequest.Builder request = HttpRequest.newBuilder(object)
.header("content-encoding", "aws-chunked")
.header("x-amz-content-sha256", mode)
.header("x-amz-date", date)
.header("x-amz-decoded-content-length", Integer.toString(body.length))
.header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope +
",SignedHeaders=" + names + ",Signature=" + seed);
status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(encoded.toByteArray()))
.build(), HttpResponse.BodyHandlers.ofByteArray()));
var read = client.send(signedUri(object, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, read);
if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Streaming upload mismatch");
status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testStreamingTrailer(HttpClient client, String base) throws Exception {
URI object = URI.create(base + "/objects/streaming-trailer-test");
byte[] body = "verified trailer payload".getBytes(StandardCharsets.UTF_8);
String date = DATE.format(Instant.now());
String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
String trailerName = "x-amz-checksum-xxhash3";
String checksum = encodedChecksum("XXHASH3", body);
String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;" +
"x-amz-decoded-content-length;x-amz-sdk-checksum-algorithm;x-amz-trailer";
String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" +
"host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode +
"\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length +
"\nx-amz-sdk-checksum-algorithm:XXHASH3\nx-amz-trailer:" + trailerName +
"\n\n" + names + "\n" + mode;
byte[] key = SigV4.signingKey(SECRET, date.substring(0, 8), REGION);
String seed = SigV4.hex(SigV4.hmac(key, "AWS4-HMAC-SHA256\n" + date + "\n" +
scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)))));
String previous = seed;
var encoded = new java.io.ByteArrayOutputStream();
for (byte[] chunk : new byte[][]{body, new byte[0]}) {
String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" +
previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" +
SigV4.hex(SigV4.hash(chunk));
previous = SigV4.hex(SigV4.hmac(key, toSign));
encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n")
.getBytes(StandardCharsets.US_ASCII));
encoded.write(chunk);
if (chunk.length != 0) encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII));
}
encoded.write((trailerName + ":" + checksum + "\r\n").getBytes(StandardCharsets.US_ASCII));
String trailerToSign = "AWS4-HMAC-SHA256-TRAILER\n" + date + "\n" + scope + "\n" +
previous + "\n" + SigV4.hex(SigV4.hash((trailerName + ":" + checksum + "\n")
.getBytes(StandardCharsets.UTF_8)));
encoded.write(("x-amz-trailer-signature=" + SigV4.hex(SigV4.hmac(key, trailerToSign)) +
"\r\n\r\n").getBytes(StandardCharsets.US_ASCII));
byte[] upload = encoded.toByteArray();
HttpRequest.Builder request = HttpRequest.newBuilder(object)
.header("content-encoding", "aws-chunked")
.header("x-amz-content-sha256", mode)
.header("x-amz-date", date)
.header("x-amz-decoded-content-length", Integer.toString(body.length))
.header("x-amz-sdk-checksum-algorithm", "XXHASH3")
.header("x-amz-trailer", trailerName)
.header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope +
",SignedHeaders=" + names + ",Signature=" + seed);
status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(upload)).build(),
HttpResponse.BodyHandlers.ofByteArray()));
var head = client.send(signedUri(object, "HEAD", new byte[0],
Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding());
status(200, head);
if (!checksum.equals(head.headers().firstValue(trailerName).orElse("")))
throw new AssertionError("Signed checksum trailer was not persisted");
byte[] tampered = upload.clone();
tampered[upload.length - 10] ^= 1;
status(400, client.send(HttpRequest.newBuilder(object)
.header("content-encoding", "aws-chunked")
.header("x-amz-content-sha256", mode)
.header("x-amz-date", date)
.header("x-amz-decoded-content-length", Integer.toString(body.length))
.header("x-amz-sdk-checksum-algorithm", "XXHASH3")
.header("x-amz-trailer", trailerName)
.header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope +
",SignedHeaders=" + names + ",Signature=" + seed)
.PUT(HttpRequest.BodyPublishers.ofByteArray(tampered)).build(),
HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testCapabilities(HttpClient client, String base) throws Exception {
URI uri = URI.create(base + "/_objectstore/capabilities");
status(403, client.send(HttpRequest.newBuilder(uri).GET().build(),
HttpResponse.BodyHandlers.ofByteArray()));
var response = client.send(signedUri(uri, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
status(200, response);
String compact = response.body().replaceAll("\\s+", "");
if (!compact.contains("\"schemaVersion\":1") ||
!compact.contains("\"service\":\"lunarsky-objectstore\"") ||
!compact.contains("\"serviceVersion\":\"" + Version.VALUE + "\"") ||
!compact.contains("\"storageMode\":\"disk\"") ||
!compact.contains("\"ListParts\"") ||
!compact.contains("\"maxObjectBytes\":1024") ||
!compact.contains("\"maxTotalBytes\":4096") ||
!compact.contains("\"maxParts\":10000"))
throw new AssertionError("Unexpected capability manifest: " + response.body());
if (!response.headers().firstValue("content-type").orElse("").startsWith("application/json") ||
!"no-store".equals(response.headers().firstValue("cache-control").orElse("")))
throw new AssertionError("Capability response headers missing");
status(400, client.send(signedUri(URI.create(uri + "?extra=1"), "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
}
private static void status(int expected, HttpResponse<?> response) {
if (response.statusCode() != expected) {
throw new AssertionError("Expected HTTP " + expected + ", got " + response.statusCode()
+ ": " + new String(response.body(), StandardCharsets.UTF_8));
+ ": " + (response.body() instanceof byte[] bytes
? new String(bytes, StandardCharsets.UTF_8) : response.body()));
}
}
@@ -167,7 +439,7 @@ public final class HttpTest {
status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/other/source")),
HttpResponse.BodyHandlers.ofByteArray()));
status(400, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/objects/source?versionId=1")),
HttpResponse.BodyHandlers.ofByteArray()));
status(501, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
@@ -180,12 +452,26 @@ public final class HttpTest {
}
private static String encodedChecksum(String algorithm, byte[] body) throws Exception {
if (algorithm.startsWith("XXHASH")) {
var checksum = new XxHashes(algorithm);
checksum.update(body, 0, body.length);
return Base64.getEncoder().encodeToString(checksum.digest());
}
if (algorithm.startsWith("CRC")) {
Checksum checksum = algorithm.equals("CRC32") ? new CRC32() : new CRC32C();
Checksum checksum = switch (algorithm) {
case "CRC32" -> new CRC32();
case "CRC32C" -> new CRC32C();
case "CRC64NVME" -> new Crc64Nvme();
default -> throw new IllegalArgumentException(algorithm);
};
checksum.update(body, 0, body.length);
long value = checksum.getValue();
return Base64.getEncoder().encodeToString(new byte[]{(byte) (value >>> 24),
(byte) (value >>> 16), (byte) (value >>> 8), (byte) value});
byte[] bytes = new byte[algorithm.equals("CRC64NVME") ? 8 : 4];
for (int i = bytes.length - 1; i >= 0; i--) {
bytes[i] = (byte) value;
value >>>= 8;
}
return Base64.getEncoder().encodeToString(bytes);
}
String name = algorithm.equals("SHA1") ? "SHA-1" :
algorithm.equals("SHA256") ? "SHA-256" :
@@ -197,7 +483,8 @@ public final class HttpTest {
URI uri = URI.create(base + "/objects/checksum-target");
byte[] body = "checksum payload".getBytes(StandardCharsets.UTF_8);
String md5 = encodedChecksum("MD5", body);
for (String algorithm : new String[]{"CRC32", "CRC32C", "SHA1", "SHA256", "SHA512", "MD5"}) {
for (String algorithm : new String[]{"CRC32", "CRC32C", "CRC64NVME", "XXHASH64",
"XXHASH3", "XXHASH128", "SHA1", "SHA256", "SHA512", "MD5"}) {
String header = "x-amz-checksum-" + algorithm.toLowerCase(java.util.Locale.ROOT);
String checksum = encodedChecksum(algorithm, body);
var stored = client.send(signedUri(uri, "PUT", body,
@@ -207,7 +494,9 @@ public final class HttpTest {
if (!checksum.equals(stored.headers().firstValue(header).orElse("")))
throw new AssertionError("Missing checksum response: " + algorithm);
var bad = client.send(signedUri(uri, "PUT", body,
Map.of(header, Base64.getEncoder().encodeToString(new byte[algorithm.startsWith("CRC") ? 4 :
Map.of(header, Base64.getEncoder().encodeToString(new byte[algorithm.equals("XXHASH128") ? 16 :
algorithm.startsWith("XXHASH") || algorithm.equals("CRC64NVME") ? 8 :
algorithm.startsWith("CRC") ? 4 :
algorithm.equals("SHA1") ? 20 : algorithm.equals("SHA256") ? 32 :
algorithm.equals("SHA512") ? 64 : 16]))), HttpResponse.BodyHandlers.ofString());
if (bad.statusCode() != 400 || !bad.body().contains("BadDigest"))
@@ -217,6 +506,13 @@ public final class HttpTest {
status(200, unchanged);
if (!java.util.Arrays.equals(body, unchanged.body()))
throw new AssertionError("Bad checksum replaced stored object");
var head = client.send(signedUri(uri, "HEAD", new byte[0],
Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding());
status(200, head);
if (!checksum.equals(head.headers().firstValue(header).orElse("")))
throw new AssertionError("Checksum was not persisted: " + algorithm);
status(400, client.send(signedUri(uri, "HEAD", new byte[0],
Map.of("x-amz-checksum-mode", "INVALID")), HttpResponse.BodyHandlers.discarding()));
}
var badMd5 = client.send(signedUri(uri, "PUT", body,
Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString());
@@ -227,8 +523,14 @@ public final class HttpTest {
status(400, client.send(signedUri(uri, "PUT", body,
Map.of("x-amz-checksum-crc32", encodedChecksum("CRC32", body),
"x-amz-sdk-checksum-algorithm", "CRC32C")), HttpResponse.BodyHandlers.ofByteArray()));
status(501, client.send(signedUri(uri, "PUT", body,
Map.of("x-amz-checksum-crc64nvme", "AAAAAAAAAAA=")), HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(uri, "PUT", body, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var automatic = client.send(signedUri(uri, "HEAD", new byte[0],
Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding());
status(200, automatic);
if (!encodedChecksum("CRC64NVME", body).equals(automatic.headers()
.firstValue("x-amz-checksum-crc64nvme").orElse("")))
throw new AssertionError("Default CRC64NVME checksum was not persisted");
status(204, client.send(signedUri(uri, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
}
@@ -296,6 +598,214 @@ public final class HttpTest {
HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testAttributes(HttpClient client, String base) throws Exception {
URI object = URI.create(base + "/objects/attributes.txt");
byte[] body = "object attributes".getBytes(StandardCharsets.UTF_8);
status(200, client.send(signedUri(object, "PUT", body, Map.of(
"x-amz-meta-project", "LunarSky", "x-amz-tagging", "kind=test&phase=one")),
HttpResponse.BodyHandlers.ofByteArray()));
var get = client.send(signedUri(object, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, get);
if (!"LunarSky".equals(get.headers().firstValue("x-amz-meta-project").orElse("")) ||
!"2".equals(get.headers().firstValue("x-amz-tagging-count").orElse("")))
throw new AssertionError("Stored attributes missing from GET");
var tagging = URI.create(object + "?tagging");
var originalTags = client.send(signedUri(URI.create(tagging + "&x-id=GetObjectTagging"),
"GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
status(200, originalTags);
if (!originalTags.body().contains("<Key>kind</Key><Value>test</Value>"))
throw new AssertionError("Object tags were not stored");
byte[] replacement = "<Tagging><TagSet><Tag><Key>stage</Key><Value>two</Value></Tag></TagSet></Tagging>"
.getBytes(StandardCharsets.UTF_8);
status(200, client.send(signedUri(tagging, "PUT", replacement, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(400, client.send(signedUri(tagging, "PUT", replacement,
Map.of("content-md5", Base64.getEncoder().encodeToString(new byte[16]))),
HttpResponse.BodyHandlers.ofByteArray()));
var replaced = client.send(signedUri(tagging, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
if (!replaced.body().contains("<Key>stage</Key><Value>two</Value>") ||
replaced.body().contains("<Key>kind</Key>")) throw new AssertionError("Tag replacement failed");
status(400, client.send(signedUri(tagging, "PUT", "<!DOCTYPE x [<!ENTITY y SYSTEM 'file:///etc/passwd'>]><Tagging/>"
.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signedUri(tagging, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var cleared = client.send(signedUri(tagging, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
if (!cleared.body().contains("<TagSet></TagSet>")) throw new AssertionError("Tag deletion failed");
status(200, client.send(signedUri(URI.create(base + "/objects/attributes-copy.txt"), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/objects/attributes.txt")),
HttpResponse.BodyHandlers.ofByteArray()));
var copied = client.send(signed(base, "GET", "attributes-copy.txt", new byte[0]),
HttpResponse.BodyHandlers.ofByteArray());
if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse("")))
throw new AssertionError("Copy lost user metadata");
status(400, client.send(signedUri(object, "PUT", body,
Map.of("x-amz-meta-bad", "a".repeat(2100))), HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testBuckets(HttpClient client, String base) throws Exception {
var root = URI.create(base + "/");
var existing = client.send(signedUri(root, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
status(200, existing);
if (!existing.body().contains("<Name>objects</Name>"))
throw new AssertionError("Default bucket absent from service listing");
var bucket = URI.create(base + "/second-bucket");
status(200, client.send(signedUri(URI.create(bucket + "?x-id=CreateBucket"),
"PUT", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(409, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(200, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var upload = client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploads"),
"POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
status(200, upload);
String uploadId = upload.body().split("<UploadId>")[1].split("</UploadId>")[0];
status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploadId=" + uploadId),
"DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
byte[] body = "second bucket".getBytes(StandardCharsets.UTF_8);
var object = URI.create(base + "/second-bucket/one.txt");
status(200, client.send(signedUri(object, "PUT", body, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var read = client.send(signedUri(object, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, read);
if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Second bucket read failed");
var crossCopy = URI.create(base + "/second-bucket/copied.txt");
status(200, client.send(signedUri(crossCopy, "PUT", new byte[0],
Map.of("x-amz-copy-source", "/objects/attributes.txt")),
HttpResponse.BodyHandlers.ofByteArray()));
var copied = client.send(signedUri(crossCopy, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse("")))
throw new AssertionError("Cross-bucket copy lost metadata");
var listed = client.send(signedUri(URI.create(base + "/second-bucket?list-type=2"),
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
if (!listed.body().contains("<Key>one.txt</Key>")) throw new AssertionError("Second bucket listing failed");
status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signedUri(crossCopy, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
status(404, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testVersioning(HttpClient client, String base) throws Exception {
URI bucket = URI.create(base + "/version-bucket");
URI configuration = URI.create(bucket + "?versioning");
URI object = URI.create(bucket + "/note.txt");
status(200, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var initial = client.send(signedUri(configuration, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
if (!initial.body().contains("<VersioningConfiguration") || initial.body().contains("<Status>"))
throw new AssertionError("New bucket versioning state");
status(200, client.send(signedUri(object, "PUT", new byte[]{1}, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
byte[] enable = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"
.getBytes(StandardCharsets.UTF_8);
status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var first = client.send(signedUri(object, "PUT", new byte[]{2}, Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, first);
String id = first.headers().firstValue("x-amz-version-id").orElseThrow();
var second = client.send(signedUri(object, "PUT", new byte[]{3}, Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, second);
if (id.equals(second.headers().firstValue("x-amz-version-id").orElseThrow()))
throw new AssertionError("Version IDs repeated");
var old = client.send(signedUri(URI.create(object + "?versionId=" + id), "GET",
new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray());
status(200, old);
if (old.body()[0] != 2) throw new AssertionError("Historical object body");
byte[] tagged = "<Tagging><TagSet><Tag><Key>kind</Key><Value>old</Value></Tag></TagSet></Tagging>"
.getBytes(StandardCharsets.UTF_8);
status(200, client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "PUT",
tagged, Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
var oldTags = client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "GET",
new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
status(200, oldTags);
if (!oldTags.body().contains("<Value>old</Value>"))
throw new AssertionError("Versioned tagging failed");
var copied = client.send(signedUri(URI.create(bucket + "/copied.txt"), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/version-bucket/note.txt?versionId=" + id)),
HttpResponse.BodyHandlers.ofByteArray());
status(200, copied);
if (!id.equals(copied.headers().firstValue("x-amz-copy-source-version-id").orElse("")))
throw new AssertionError("Copy did not report source version");
var copyBody = client.send(signedUri(URI.create(bucket + "/copied.txt"), "GET",
new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray());
status(200, copyBody);
if (copyBody.body()[0] != 2) throw new AssertionError("Copy of old version failed");
var deleted = client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(204, deleted);
String marker = deleted.headers().firstValue("x-amz-version-id").orElseThrow();
var hidden = client.send(signedUri(object, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(404, hidden);
if (!"true".equals(hidden.headers().firstValue("x-amz-delete-marker").orElse("")))
throw new AssertionError("Missing delete marker response header");
status(405, client.send(signedUri(URI.create(object + "?versionId=" + marker), "GET",
new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
var listed = client.send(signedUri(URI.create(bucket + "?versions&max-keys=2"), "GET",
new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
status(200, listed);
if (!listed.body().contains("<DeleteMarker>") || !listed.body().contains("<IsTruncated>true</IsTruncated>"))
throw new AssertionError("Version listing did not include delete marker");
status(204, client.send(signedUri(URI.create(object + "?versionId=" + marker), "DELETE",
new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
var restored = client.send(signedUri(object, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, restored);
if (restored.body()[0] != 3) throw new AssertionError("Deleting marker did not restore current version");
byte[] suspend = "<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>"
.getBytes(StandardCharsets.UTF_8);
status(200, client.send(signedUri(configuration, "PUT", suspend, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var nullVersion = client.send(signedUri(object, "PUT", new byte[]{4}, Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, nullVersion);
if (!"null".equals(nullVersion.headers().firstValue("x-amz-version-id").orElse("")))
throw new AssertionError("Suspended write did not produce null version");
status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
URI multipartObject = URI.create(bucket + "/multipart.txt");
var initiated = client.send(signedUri(URI.create(multipartObject + "?uploads"), "POST",
new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
status(200, initiated);
String uploadId = initiated.body().split("<UploadId>")[1].split("</UploadId>")[0];
byte[] partBody = "versioned multipart".getBytes(StandardCharsets.UTF_8);
var part = client.send(signedUri(URI.create(multipartObject + "?partNumber=1&uploadId=" + uploadId),
"PUT", partBody, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
status(200, part);
String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
part.headers().firstValue("etag").orElseThrow() + "</ETag></Part></CompleteMultipartUpload>";
var completed = client.send(signedUri(URI.create(multipartObject + "?uploadId=" + uploadId),
"POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, completed);
String multipartVersion = completed.headers().firstValue("x-amz-version-id").orElseThrow();
status(204, client.send(signedUri(multipartObject, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
var retainedMultipart = client.send(signedUri(URI.create(multipartObject + "?versionId=" +
multipartVersion), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray());
status(200, retainedMultipart);
if (!java.util.Arrays.equals(partBody, retainedMultipart.body()))
throw new AssertionError("Completed multipart version was not retained");
}
public static void main(String[] args) throws Exception {
Path root = Files.createTempDirectory("store-http-test-");
var executor = Executors.newVirtualThreadPerTaskExecutor();
@@ -303,19 +813,28 @@ public final class HttpTest {
DiskStore store = new DiskStore(root, 1024, 4096);
try {
var app = new Main(store,
new SigV4(ACCESS, SECRET, REGION, Clock.systemUTC()), "objects");
new SigV4(Map.of(ACCESS, SECRET, SECONDARY, SECONDARY_SECRET),
ACCESS, REGION, Clock.systemUTC()), "objects");
server.setExecutor(executor);
server.createContext("/", app::handle);
server.start();
String base = "http://127.0.0.1:" + server.getAddress().getPort();
HttpClient client = HttpClient.newHttpClient();
testCapabilities(client, base);
testPresigned(client, base);
testStreaming(client, base);
testStreamingTrailer(client, base);
testObjects(client, base);
testListing(client, base);
testCopy(client, base);
testChecksums(client, base);
testMultipart(client, base);
testAttributes(client, base);
testDelete(client, base);
System.out.println("HTTP tests passed: objects, copy, checksums, listing, multipart");
testBuckets(client, base);
testVersioning(client, base);
testAcl(client, base);
System.out.println("HTTP tests passed: capabilities, objects, copy, checksums, listing, multipart, attributes, buckets, versioning, ACLs");
} finally {
server.stop(0);
executor.close();
test/cloud/lunarsky/store/StoreTest.java
+284
View File
@@ -5,6 +5,7 @@ import java.util.Arrays;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.util.List;
import java.util.Map;
public final class StoreTest {
interface Operation {void run() throws Exception;}
@@ -36,9 +37,87 @@ public final class StoreTest {
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers));
headers.add("host","duplicate");
fails(403,()->auth.verify("GET",uri,headers));
var presignedHeaders = new com.sun.net.httpserver.Headers();
presignedHeaders.set("host", "examplebucket.s3.amazonaws.com");
var presigned = java.net.URI.create("/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256" +
"&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request" +
"&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host" +
"&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404");
if (!"UNSIGNED-PAYLOAD".equals(auth.verifyRequest("GET", presigned, presignedHeaders).payload()))
throw new AssertionError("Official presigned URL was not accepted");
fails(403, () -> auth.verifyRequest("PUT", presigned, presignedHeaders));
fails(403, () -> new SigV4("AKIAIOSFODNN7EXAMPLE",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1",
java.time.Clock.fixed(java.time.Instant.parse("2013-05-25T00:00:01Z"),
java.time.ZoneOffset.UTC)).verifyRequest("GET", presigned, presignedHeaders));
System.out.println("SigV4 official vector and tampering tests passed");
}
private static void testAwsChunked() throws Exception {
String secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
String date = "20130524T000000Z";
String scope = "20130524/us-east-1/s3/aws4_request";
byte[] key = SigV4.signingKey(secret, "20130524", "us-east-1");
byte[] body = new byte[66560];
Arrays.fill(body, (byte) 'a');
var encoded = new ByteArrayOutputStream();
encoded.write("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n".getBytes());
encoded.write(body, 0, 65536);
encoded.write("\r\n400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n".getBytes());
encoded.write(body, 65536, 1024);
encoded.write("\r\n0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n".getBytes());
var authorization = new SigV4.Verified("STREAMING-AWS4-HMAC-SHA256-PAYLOAD", "", key,
date, scope, "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9");
try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(encoded.toByteArray()),
authorization, body.length, null)) {
if (!Arrays.equals(body, stream.readAllBytes())) throw new AssertionError("Signed chunk vector mismatch");
}
byte[] tampered = encoded.toByteArray();
tampered[100] = 'b';
fails(400, () -> {
try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(tampered),
authorization, body.length, null)) { stream.readAllBytes(); }
});
var withTrailer = new ByteArrayOutputStream();
withTrailer.write("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n".getBytes());
withTrailer.write(body, 0, 65536);
withTrailer.write("\r\n400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n".getBytes());
withTrailer.write(body, 65536, 1024);
withTrailer.write("\r\n0;chunk-signature=2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992\r\n".getBytes());
withTrailer.write("x-amz-checksum-crc32c:sOO8/Q==\r\n".getBytes());
withTrailer.write("x-amz-trailer-signature=d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435\r\n\r\n".getBytes());
var trailerAuthorization = new SigV4.Verified("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER",
"", key, date, scope, "106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e");
try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(withTrailer.toByteArray()),
trailerAuthorization, body.length, "x-amz-checksum-crc32c")) {
if (!Arrays.equals(body, stream.readAllBytes()) || !"sOO8/Q==".equals(stream.trailerValue()))
throw new AssertionError("Signed checksum trailer vector mismatch");
}
}
private static void testCrc64Nvme() {
byte[] zeros = new byte[32];
Crc64Nvme checksum = new Crc64Nvme();
checksum.update(zeros, 0, zeros.length);
if (checksum.getValue() != 0xcf3473434d4ecf3bL)
throw new AssertionError("CRC64NVME test vector mismatch");
}
private static void testXxHashes() {
byte[] body = "abc".getBytes(java.nio.charset.StandardCharsets.US_ASCII);
Map<String, String> vectors = Map.of(
"XXHASH64", "44bc2cf5ad770999",
"XXHASH3", "78af5f94892f3950",
"XXHASH128", "06b05ab6733a618578af5f94892f3950");
for (var entry : vectors.entrySet()) {
XxHashes hash = new XxHashes(entry.getKey());
hash.update(body, 0, 1);
hash.update(body, 1, 2);
if (!SigV4.hex(hash.digest()).equals(entry.getValue()))
throw new AssertionError(entry.getKey() + " test vector mismatch");
}
}
private static void testInitialStore(Path root) throws Exception {
try(var store=new DiskStore(root,8,10)){
byte[] body={1,2,3,4,5,6};
@@ -134,14 +213,219 @@ public final class StoreTest {
try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
}
private static void testAttributesRestart(Path root) throws Exception {
Path data = root.resolve("attributes");
byte[] body = {1, 2, 3};
var crc = new Crc64Nvme();
crc.update(body, 0, body.length);
byte[] digest = ByteBuffer.allocate(8).putLong(crc.getValue()).array();
Map<String, String> checksums = Map.of("x-amz-checksum-crc64nvme",
java.util.Base64.getEncoder().encodeToString(digest));
try (var store = new DiskStore(data, 8, 10)) {
store.put("test", "metadata", new ByteArrayInputStream(body), body.length,
SigV4.hex(SigV4.hash(body)), null, false, "text/plain",
Map.of("project", "LunarSky"), Map.of("stage", "one"), () -> checksums);
store.setTags("test", "metadata", Map.of("stage", "two"));
}
try (var store = new DiskStore(data, 8, 10);
var object = store.open("test", "metadata")) {
if (!Arrays.equals(body, object.stream().readAllBytes()) ||
!object.metadata().userMetadata().equals(Map.of("project", "LunarSky")) ||
!object.metadata().tags().equals(Map.of("stage", "two")) ||
!object.metadata().checksums().equals(checksums))
throw new AssertionError("Object attributes were lost after restart");
}
}
private static void testV3Record(Path root) throws Exception {
Path data = root.resolve("v3-record");
String bucket = "test", key = "v3-object";
byte[] body = {8, 9, 10};
byte[] bucketBytes = bucket.getBytes(java.nio.charset.StandardCharsets.UTF_8);
byte[] keyBytes = key.getBytes(java.nio.charset.StandardCharsets.UTF_8);
byte[] typeBytes = "text/plain".getBytes(java.nio.charset.StandardCharsets.UTF_8);
byte[] metadata = ObjectAttributes.encode(Map.of("legacy", "yes"), 4096);
byte[] tags = ObjectAttributes.encode(Map.of("source", "v3"), 8192);
String pathHash = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(java.nio.charset.StandardCharsets.UTF_8)));
Path path = data.resolve("objects").resolve(pathHash.substring(0, 2)).resolve(pathHash);
Files.createDirectories(path.getParent());
ByteBuffer bytes = ByteBuffer.allocate(82 + bucketBytes.length + keyBytes.length +
typeBytes.length + metadata.length + tags.length + body.length);
bytes.putLong(0x4c534f424a303033L).putLong(body.length).putLong(123456789L)
.put(MessageDigest.getInstance("MD5").digest(body)).put(SigV4.hash(body))
.putShort((short) bucketBytes.length).putShort((short) keyBytes.length)
.putShort((short) typeBytes.length).putShort((short) metadata.length)
.putShort((short) tags.length).put(bucketBytes).put(keyBytes).put(typeBytes)
.put(metadata).put(tags).put(body);
Files.write(path, bytes.array());
try (var store = new DiskStore(data, 64, 128)) {
try (var object = store.open(bucket, key)) {
if (!Arrays.equals(body, object.stream().readAllBytes()) ||
!object.metadata().userMetadata().equals(Map.of("legacy", "yes")) ||
!object.metadata().tags().equals(Map.of("source", "v3")))
throw new AssertionError("V3 record was not readable");
}
store.setTags(bucket, key, Map.of("source", "v4"));
}
try (var store = new DiskStore(data, 64, 128);
var object = store.open(bucket, key)) {
if (!Arrays.equals(body, object.stream().readAllBytes()) ||
!object.metadata().tags().equals(Map.of("source", "v4")))
throw new AssertionError("V3 record upgrade failed");
}
}
private static void testBucketRestart(Path root) throws Exception {
Path data = root.resolve("buckets");
byte[] body = {4, 5, 6};
try (var store = new DiskStore(data, 8, 10)) {
store.ensureBucket("default-bucket");
store.createBucket("second-bucket");
store.put("second-bucket", "object", new ByteArrayInputStream(body), body.length,
SigV4.hex(SigV4.hash(body)), null, false, "application/octet-stream");
}
try (var store = new DiskStore(data, 8, 10);
var object = store.open("second-bucket", "object")) {
if (store.buckets().size() != 2 || !Arrays.equals(body, object.stream().readAllBytes()))
throw new AssertionError("Bucket catalog was lost after restart");
fails(409, () -> store.deleteBucket("second-bucket"));
store.delete("second-bucket", "object");
store.deleteBucket("second-bucket");
fails(404, () -> store.bucket("second-bucket"));
}
}
private static void testAclPersistence(Path root) throws Exception {
Path data = root.resolve("acl");
Map<String, String> grant = Map.of("SECONDARYKEY1234", Integer.toString(Acl.READ));
byte[] body = {9, 8, 7};
String older;
try (var store = new DiskStore(data, 32, 128)) {
store.ensureBucket("acl-bucket");
store.setBucketAcl("acl-bucket", grant);
store.setVersioning("acl-bucket", ObjectStorage.VersioningState.ENABLED);
older = store.put("acl-bucket", "image", new ByteArrayInputStream(body), body.length,
SigV4.hex(SigV4.hash(body)), null, false, "image/png",
Map.of(), Map.of(), Map::of, grant).versionId();
store.put("acl-bucket", "image", new ByteArrayInputStream(body), body.length,
SigV4.hex(SigV4.hash(body)), null, false, "image/png");
}
try (var store = new DiskStore(data, 32, 128);
var previous = store.open("acl-bucket", "image", older);
var current = store.open("acl-bucket", "image")) {
if (!store.bucket("acl-bucket").acl().equals(grant) ||
!previous.metadata().acl().equals(grant) || !current.metadata().acl().isEmpty())
throw new AssertionError("ACL grants changed after restart or version replacement");
store.setObjectAcl("acl-bucket", "image", older, Map.of(Acl.ALL_USERS, "1"));
}
try (var store = new DiskStore(data, 32, 128);
var previous = store.open("acl-bucket", "image", older)) {
if (!previous.metadata().acl().equals(Map.of(Acl.ALL_USERS, "1")))
throw new AssertionError("Version-specific ACL edit was not durable");
}
}
private static void testAdditionalKeys(Path root) throws Exception {
Path file = root.resolve("access-keys");
Files.writeString(file, "SECONDARYKEY1234:secondary-secret-key-that-is-at-least-32-characters\n");
Map<String, String> keys = Main.identities(
Map.of("S3_CREDENTIALS_FILE", file.toString()),
"TESTACCESSKEY123", "test-secret-key-that-is-at-least-32-characters");
if (keys.size() != 2 || !keys.containsKey("SECONDARYKEY1234"))
throw new AssertionError("Additional access key was not loaded");
Files.writeString(file, "TESTACCESSKEY123:duplicate-root-secret-that-is-at-least-32-characters\n");
try {
Main.identities(Map.of("S3_CREDENTIALS_FILE", file.toString()),
"TESTACCESSKEY123", "test-secret-key-that-is-at-least-32-characters");
throw new AssertionError("Duplicate root key was accepted");
} catch (IllegalArgumentException expected) { }
}
private static void testVersioning(Path root) throws Exception {
Path data = root.resolve("versioning");
String first;
String second;
String marker;
try (var store = new DiskStore(data, 8, 100)) {
store.createBucket("versioned-bucket");
byte[] old = {1};
store.put("versioned-bucket", "note", new ByteArrayInputStream(old), old.length,
SigV4.hex(SigV4.hash(old)), null, false, "text/plain");
store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.ENABLED);
byte[] newer = {2};
first = store.put("versioned-bucket", "note", new ByteArrayInputStream(newer), newer.length,
SigV4.hex(SigV4.hash(newer)), null, false, "text/plain").versionId();
byte[] latest = {3};
second = store.put("versioned-bucket", "note", new ByteArrayInputStream(latest), latest.length,
SigV4.hex(SigV4.hash(latest)), null, false, "text/plain").versionId();
if (first == null || second == null || first.equals(second)) throw new AssertionError("Unique versions");
if (store.usedBytes() != 3) throw new AssertionError("Retained versions did not count toward capacity");
try (var object = store.open("versioned-bucket", "note", first)) {
if (object.stream().read() != 2) throw new AssertionError("Old version was overwritten");
}
marker = store.delete("versioned-bucket", "note", null).versionId();
fails(404, () -> store.open("versioned-bucket", "note"));
if (!store.list("versioned-bucket", "", "", 10, null).objects().isEmpty())
throw new AssertionError("Delete marker appeared in current listing");
var page = store.listVersions("versioned-bucket", "", null, null, 2);
if (!page.truncated() || !page.entries().getFirst().deleteMarker() ||
!page.entries().get(1).versionId().equals(second)) throw new AssertionError("Version listing");
var rest = store.listVersions("versioned-bucket", "", page.nextKey(), page.nextVersionId(), 10);
if (rest.entries().size() != 2 || !rest.entries().getLast().versionId().equals("null"))
throw new AssertionError("Version pagination");
store.delete("versioned-bucket", "note", marker);
try (var object = store.open("versioned-bucket", "note")) {
if (object.stream().read() != 3) throw new AssertionError("Delete marker removal");
}
store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.SUSPENDED);
byte[] suspended = {4};
var nullVersion = store.put("versioned-bucket", "note", new ByteArrayInputStream(suspended),
suspended.length, SigV4.hex(SigV4.hash(suspended)), null, false, "text/plain");
if (!"null".equals(nullVersion.versionId())) throw new AssertionError("Suspended null version");
if (store.usedBytes() != 3) throw new AssertionError("Suspended write did not replace null version");
try (var object = store.open("versioned-bucket", "note", second)) {
if (object.stream().read() != 3) throw new AssertionError("Suspension removed a version");
}
store.setTags("versioned-bucket", "note", Map.of("stage", "suspended"));
}
try (var store = new DiskStore(data, 8, 100)) {
if (store.bucket("versioned-bucket").versioning() != ObjectStorage.VersioningState.SUSPENDED)
throw new AssertionError("Versioning state was lost");
try (var object = store.open("versioned-bucket", "note")) {
if (object.stream().read() != 4 || !object.metadata().tags().equals(Map.of("stage", "suspended")))
throw new AssertionError("Versioned object was lost");
}
store.delete("versioned-bucket", "note");
fails(404, () -> store.open("versioned-bucket", "note"));
if (store.usedBytes() != 2) throw new AssertionError("Suspended delete did not release null version");
try (var object = store.open("versioned-bucket", "note", second)) {
if (object.stream().read() != 3) throw new AssertionError("Suspended delete removed old version");
}
fails(409, () -> store.deleteBucket("versioned-bucket"));
for (var entry : store.listVersions("versioned-bucket", "", null, null, 10).entries())
store.delete("versioned-bucket", "note", entry.versionId());
if (store.usedBytes() != 0) throw new AssertionError("Version deletes did not release capacity");
store.deleteBucket("versioned-bucket");
}
}
public static void main(String[] args) throws Exception {
testSignature();
testAwsChunked();
testCrc64Nvme();
testXxHashes();
Path root = Files.createTempDirectory("store-test-");
try {
testInitialStore(root);
testRestart(root);
testMultipartRecovery(root);
testLegacyRecord(root);
testAttributesRestart(root);
testV3Record(root);
testBucketRestart(root);
testAclPersistence(root);
testAdditionalKeys(root);
testVersioning(root);
testCorruption(root);
System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete");
} finally {