Release ObjectStore 0.0.2

test/cloud/lunarsky/store/CliTest.java

@@ -0,0 +1,54 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.PrintStream;
+import java.io.RandomAccessFile;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+
+public final class CliTest {
+    public static void main(String[] args) throws Exception {
+        Path root = Files.createTempDirectory("store-cli-test-");
+        try {
+            var versionOutput = new ByteArrayOutputStream();
+            if (Cli.run(new String[]{"version"}, root, new PrintStream(versionOutput), System.err) != 0 ||
+                !versionOutput.toString().contains(Version.VALUE)) throw new AssertionError("CLI version");
+            byte[] payload = "verified".getBytes(StandardCharsets.UTF_8);
+            try (var store = new DiskStore(root, 100, 1000)) {
+                store.put("objects", "example", new ByteArrayInputStream(payload), payload.length,
+                    SigV4.hex(SigV4.hash(payload)), null, false, "text/plain");
+                var output = new ByteArrayOutputStream();
+                int status = Cli.run(new String[]{"status"}, root, new PrintStream(output), System.err);
+                if (status != 0 || !output.toString().contains("objects=1") ||
+                    !output.toString().contains("payload_bytes=8")) throw new AssertionError("CLI status");
+                output.reset();
+                status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), System.err);
+                if (status != 0 || !output.toString().contains("verified_objects=1"))
+                    throw new AssertionError("CLI verification");
+            }
+            String id = SigV4.hex(SigV4.hash("objects/example".getBytes(StandardCharsets.UTF_8)));
+            Path file = root.resolve("objects").resolve(id.substring(0, 2)).resolve(id);
+            int header;
+            try (var input = new DataInputStream(Files.newInputStream(file))) {
+                header = DiskStore.readRecord(input).headerLength();
+            }
+            try (var bytes = new RandomAccessFile(file.toFile(), "rw")) {
+                bytes.seek(header);
+                bytes.write('X');
+            }
+            var output = new ByteArrayOutputStream();
+            int status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), new PrintStream(output));
+            if (status != 1 || !output.toString().contains("Object checksum mismatch"))
+                throw new AssertionError("CLI missed corrupted payload");
+            System.out.println("CLI tests passed: version, live status, verification, corruption exit code");
+        } finally {
+            try (var paths = Files.walk(root)) {
+                for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path);
+            }
+        }
+    }
+}

test/cloud/lunarsky/store/ClusterIntegrationTest.java

@@ -0,0 +1,144 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Map;
+
+public final class ClusterIntegrationTest {
+    private static final String KEY = "cluster-test/survivor";
+    public static void main(String[] args) throws Exception {
+        Map<String, String> env = System.getenv();
+        String[] urls = env.get("CLUSTER_NODES").split(",");
+        try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"),
+                env.get("POSTGRES_PASSWORD"), env.get("S3_BUCKET"),
+                Arrays.stream(urls).map(URI::create).toList(), env.get("CLUSTER_TOKEN"), null,
+                134217728, 2147483648L,
+                !"same-host".equals(args[0]) && "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS")))) {
+            String bucket = env.get("S3_BUCKET");
+            byte[] stable = "acknowledged object survives node loss".getBytes(StandardCharsets.UTF_8);
+            switch (args[0]) {
+                case "basic" -> {
+                    byte[] large = new byte[ClusterNode.MAX_SEGMENT + 37];
+                    for (int i = 0; i < large.length; i++) large[i] = (byte) (i * 31);
+                    String largeKey = "cluster-test/large";
+                    put(store, bucket, largeKey, large, false);
+                    try (var opened = store.open(bucket, largeKey)) {
+                        require(opened.metadata().length() == large.length, "Wrong object length");
+                        require(Arrays.equals(large, opened.stream().readAllBytes()), "Multi-segment read mismatch");
+                    }
+                    require(store.list(bucket, "cluster-test/", "", 10, null).keyCount() >= 1, "Listing missed object");
+                    try {
+                        put(store, bucket, largeKey, large, true);
+                        throw new AssertionError("Create-only overwrite succeeded");
+                    } catch (StoreException error) { require(error.status == 412, "Wrong create-only status"); }
+                    put(store, bucket, largeKey, "replacement".getBytes(StandardCharsets.UTF_8), false);
+                    try (var opened = store.open(bucket, largeKey)) {
+                        require("replacement".equals(new String(opened.stream().readAllBytes(), StandardCharsets.UTF_8)),
+                            "Overwrite was not visible");
+                    }
+                    store.delete(bucket, largeKey);
+                    try { store.open(bucket, largeKey); throw new AssertionError("Deleted object remained visible"); }
+                    catch (StoreException error) { require(error.status == 404, "Wrong missing-object status"); }
+                    put(store, bucket, KEY, stable, false);
+                    require(store.ready(), "Healthy cluster is not ready");
+                    System.out.println("Cluster basic test passed");
+                }
+                case "degraded" -> {
+                    require(store.ready(), "Two available nodes should be ready");
+                    try (var opened = store.open(bucket, KEY)) {
+                        require(Arrays.equals(stable, opened.stream().readAllBytes()), "Acknowledged object was lost");
+                    }
+                    byte[] value = "written with one node down".getBytes(StandardCharsets.UTF_8);
+                    put(store, bucket, "cluster-test/degraded", value, false);
+                    try (var opened = store.open(bucket, "cluster-test/degraded")) {
+                        require(Arrays.equals(value, opened.stream().readAllBytes()), "Degraded write was not readable");
+                    }
+                    System.out.println("Cluster degraded test passed");
+                }
+                case "quorum-lost" -> {
+                    require(!store.ready(), "One available node must not be ready");
+                    try {
+                        put(store, bucket, "cluster-test/rejected", new byte[]{1}, false);
+                        throw new AssertionError("Write succeeded with only one node");
+                    } catch (StoreException error) { require(error.status == 503, "Wrong unavailable status"); }
+                    try { store.open(bucket, "cluster-test/rejected"); throw new AssertionError("Failed write became visible"); }
+                    catch (StoreException error) { require(error.status == 404, "Partial object became visible"); }
+                    System.out.println("Cluster quorum-loss test passed");
+                }
+                case "recovered" -> {
+                    require(store.ready(), "Restarted cluster is not ready");
+                    try (var opened = store.open(bucket, KEY)) {
+                        require(Arrays.equals(stable, opened.stream().readAllBytes()), "Object lost across restart");
+                    }
+                    System.out.println("Cluster recovery test passed");
+                }
+                case "concurrent" -> {
+                    String key = "cluster-test/concurrent";
+                    byte[] first = "concurrent-first".getBytes(StandardCharsets.UTF_8);
+                    byte[] second = "concurrent-second".getBytes(StandardCharsets.UTF_8);
+                    var start = new java.util.concurrent.CountDownLatch(1);
+                    var executor = java.util.concurrent.Executors.newFixedThreadPool(2);
+                    try {
+                        var a = executor.submit(() -> {
+                            start.await(); put(store, bucket, key, first, false); return null;
+                        });
+                        var b = executor.submit(() -> {
+                            start.await(); put(store, bucket, key, second, false); return null;
+                        });
+                        start.countDown();
+                        a.get(); b.get();
+                        try (var opened = store.open(bucket, key)) {
+                            byte[] actual = opened.stream().readAllBytes();
+                            require(Arrays.equals(actual, first) || Arrays.equals(actual, second),
+                                "Concurrent PUT produced a partial object");
+                        }
+                        require(store.list(bucket, key, "", 10, null).keyCount() == 1,
+                            "Concurrent PUT produced duplicate key entries");
+                    } finally { executor.shutdownNow(); }
+                    System.out.println("Cluster concurrent overwrite test passed");
+                }
+                case "same-host" -> {
+                    require(!store.ready(), "Containers on one physical host must not form a storage quorum");
+                    try {
+                        put(store, bucket, "cluster-test/same-host-rejected", new byte[]{1}, false);
+                        throw new AssertionError("Write succeeded without two physical storage hosts");
+                    } catch (StoreException error) { require(error.status == 503, "Wrong same-host rejection status"); }
+                    System.out.println("Same-host replicas correctly fail the physical-host quorum");
+                }
+                case "joined" -> {
+                    require(urls.length == 4, "Expansion test requires four registered nodes");
+                    var newNode = NodeClient.probe(URI.create(urls[3]), env.get("CLUSTER_TOKEN"));
+                    for (int i = 0; i < 32; i++) {
+                        String key = "cluster-test/expanded-" + i;
+                        byte[] value = ("expanded object " + i).getBytes(StandardCharsets.UTF_8);
+                        put(store, bucket, key, value, false);
+                        try (var opened = store.open(bucket, key)) {
+                            require(Arrays.equals(value, opened.stream().readAllBytes()), "Expanded object was not readable");
+                        }
+                    }
+                    try (var connection = java.sql.DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                            env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
+                         var query = connection.prepareStatement(
+                             "SELECT count(*) FROM cluster_segments WHERE ? = ANY(replica_ids)")) {
+                        query.setObject(1, newNode.nodeId());
+                        try (var result = query.executeQuery()) {
+                            result.next();
+                            require(result.getLong(1) > 0, "Joined node received no new segments");
+                        }
+                    }
+                    System.out.println("Joined node accepted new placements while previous objects stayed readable");
+                }
+                default -> throw new IllegalArgumentException("Unknown test phase");
+            }
+        }
+    }
+    private static void put(ClusterStore store, String bucket, String key, byte[] data, boolean createOnly) throws Exception {
+        store.put(bucket, key, new ByteArrayInputStream(data), data.length, SigV4.hex(SigV4.hash(data)),
+            null, createOnly, "application/octet-stream");
+    }
+    private static void require(boolean condition, String message) {
+        if (!condition) throw new AssertionError(message);
+    }
+}

test/cloud/lunarsky/store/ClusterMigrationTest.java

@@ -0,0 +1,89 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.sql.DriverManager;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public final class ClusterMigrationTest {
+    private static final String KEY = "migration/legacy-object";
+    private static final byte[] DATA = "legacy object survives stable-node migration".getBytes(StandardCharsets.UTF_8);
+
+    public static void main(String[] args) throws Exception {
+        Map<String, String> env = System.getenv();
+        List<URI> urls = Arrays.stream(env.get("CLUSTER_NODES").split(","))
+            .map(URI::create).toList();
+        String token = env.get("CLUSTER_TOKEN");
+        List<NodeClient.Node> addresses = new ArrayList<>();
+        for (URI url : urls) {
+            NodeIdentity identity = NodeClient.probe(url, token);
+            addresses.add(new NodeClient.Node(identity.nodeId(), identity.hostId(), url));
+        }
+        NodeClient nodes = new NodeClient(addresses, token, null);
+        String bucket = env.get("S3_BUCKET");
+        if (args[0].equals("create")) {
+            UUID segment = UUID.randomUUID();
+            byte[] hash = SigV4.hash(DATA);
+            for (int i = 0; i < nodes.count(); i++) nodes.put(i, segment, DATA, hash);
+            try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                    env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
+                 var statement = connection.createStatement()) {
+                statement.execute("CREATE TABLE cluster_usage (bucket text PRIMARY KEY, used_bytes bigint NOT NULL)");
+                statement.execute("CREATE TABLE cluster_objects (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, length bigint NOT NULL, modified bigint NOT NULL, etag text NOT NULL, sha256 bytea NOT NULL, content_type text NOT NULL, PRIMARY KEY (bucket, object_key))");
+                statement.execute("CREATE TABLE cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))");
+                statement.execute("CREATE TABLE cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))");
+                try (var insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, ?)")) {
+                    insert.setString(1, bucket); insert.setLong(2, DATA.length); insert.executeUpdate();
+                }
+                UUID generation = UUID.randomUUID();
+                try (var insert = connection.prepareStatement("INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) {
+                    insert.setString(1, bucket); insert.setString(2, KEY); insert.setObject(3, generation);
+                    insert.setLong(4, DATA.length); insert.setLong(5, System.currentTimeMillis());
+                    insert.setString(6, HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(DATA)));
+                    insert.setBytes(7, hash); insert.setString(8, "text/plain"); insert.executeUpdate();
+                }
+                try (var insert = connection.prepareStatement("INSERT INTO cluster_segments VALUES (?, 0, ?, ?, ?, '0,1,2')")) {
+                    insert.setObject(1, generation); insert.setObject(2, segment);
+                    insert.setInt(3, DATA.length); insert.setBytes(4, hash); insert.executeUpdate();
+                }
+            }
+            System.out.println("Legacy cluster fixture created");
+            return;
+        }
+        if (!args[0].equals("verify")) throw new IllegalArgumentException("Use create or verify");
+        List<URI> reordered = new ArrayList<>(urls);
+        java.util.Collections.reverse(reordered);
+        try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"),
+                env.get("POSTGRES_PASSWORD"), bucket, reordered, token, null,
+                134217728, 2147483648L, true)) {
+            try (var opened = store.open(bucket, KEY)) {
+                if (!Arrays.equals(DATA, opened.stream().readAllBytes()))
+                    throw new AssertionError("Migrated object changed after node URL reorder");
+            }
+            store.put(bucket, "migration/new-object", new ByteArrayInputStream(DATA), DATA.length,
+                SigV4.hex(SigV4.hash(DATA)), null, false, "text/plain");
+        }
+        UUID segment;
+        try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
+             var statement = connection.createStatement();
+             var result = statement.executeQuery("SELECT segment_id FROM cluster_segments WHERE replicas='0,1,2' LIMIT 1")) {
+            if (!result.next()) throw new AssertionError("Legacy segment missing after migration");
+            segment = (UUID) result.getObject(1);
+        }
+        NodeClient wrong = new NodeClient(List.of(new NodeClient.Node(addresses.get(0).id(),
+            addresses.get(0).hostId(), addresses.get(1).url())), token, null);
+        try {
+            wrong.get(0, segment, DATA.length, SigV4.hash(DATA));
+            throw new AssertionError("Node swap was not rejected");
+        } catch (java.io.IOException expected) { }
+        System.out.println("Stable node migration, reordered gateway config, and wrong-node rejection passed");
+    }
+}

test/cloud/lunarsky/store/ClusterNodeTest.java

@@ -0,0 +1,124 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.UUID;
+
+public final class ClusterNodeTest {
+    public static void main(String[] args) throws Exception {
+        Path root = Files.createTempDirectory("objectstore-node-");
+        String token = "local-cluster-test-token-0123456789";
+        String repairToken = "local-repair-test-token-0123456789";
+        UUID id = UUID.randomUUID();
+        UUID hostId = UUID.randomUUID();
+        UUID nodeId;
+        byte[] value = "a durable segment".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        try (ClusterNode node = new ClusterNode(root, token, repairToken, hostId)) {
+            try {
+                new ClusterNode(root, token, repairToken, hostId);
+                throw new AssertionError("Second writer opened a locked node directory");
+            } catch (IOException expected) { }
+            HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+            server.createContext("/", node::handle);
+            server.start();
+            try {
+                URI uri = URI.create("http://127.0.0.1:" + server.getAddress().getPort());
+                NodeIdentity identity = NodeClient.probe(uri, token);
+                nodeId = identity.nodeId();
+                require(identity.hostId().equals(hostId), "Node reported the wrong storage host");
+                NodeClient client = new NodeClient(List.of(new NodeClient.Node(identity.nodeId(), hostId, uri)),
+                    token, repairToken);
+                client.put(0, id, value, SigV4.hash(value));
+                client.put(0, id, value, SigV4.hash(value));
+                require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))), "Roundtrip failed");
+                var request = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).timeout(Duration.ofSeconds(3))
+                    .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString())
+                    .header("X-Cluster-Sha256", SigV4.hex(SigV4.hash(value)))
+                    .PUT(HttpRequest.BodyPublishers.ofString("wrong")).build();
+                var response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
+                require(response.statusCode() == 400, "Wrong checksum accepted");
+                var unauthenticated = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).GET().build();
+                require(HttpClient.newHttpClient().send(unauthenticated, HttpResponse.BodyHandlers.ofString())
+                    .statusCode() == 403, "Unauthenticated read accepted");
+                var wrongNode = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).timeout(Duration.ofSeconds(3))
+                    .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", UUID.randomUUID().toString())
+                    .GET().build();
+                require(HttpClient.newHttpClient().send(wrongNode, HttpResponse.BodyHandlers.ofString())
+                    .statusCode() == 409, "Wrong-node request was accepted");
+                Path segment = root.resolve("segments").resolve(id.toString().substring(0, 2)).resolve(id.toString());
+                Files.writeString(segment, "corrupted");
+                try {
+                    client.get(0, id, value.length, SigV4.hash(value));
+                    throw new AssertionError("Corrupted replica passed verification");
+                } catch (IOException expected) { }
+                NodeClient gateway = new NodeClient(List.of(new NodeClient.Node(identity.nodeId(), hostId, uri)),
+                    token, null);
+                try {
+                    gateway.repair(0, id, value, SigV4.hash(value));
+                    throw new AssertionError("Gateway was allowed to repair a replica");
+                } catch (IOException expected) { }
+                var forgedRepair = HttpRequest.newBuilder(uri.resolve("/segments/" + id))
+                    .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString())
+                    .header("X-Cluster-Repair", "true").header("X-Cluster-Sha256", SigV4.hex(SigV4.hash(value)))
+                    .PUT(HttpRequest.BodyPublishers.ofByteArray(value)).build();
+                require(HttpClient.newHttpClient().send(forgedRepair, HttpResponse.BodyHandlers.discarding())
+                    .statusCode() == 403, "Shared node token was allowed to repair a replica");
+                client.repair(0, id, value, SigV4.hash(value));
+                require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))),
+                    "Repair did not restore the original bytes");
+            } finally { server.stop(0); }
+        }
+        Path pending = root.resolve("pending").resolve("unfinished.part");
+        Files.writeString(pending, "unfinished");
+        try (ClusterNode restarted = new ClusterNode(root, token, repairToken, hostId)) {
+            require(!Files.exists(pending), "Incomplete segment survived restart");
+            Path segment = root.resolve("segments").resolve(id.toString().substring(0, 2)).resolve(id.toString());
+            require(java.util.Arrays.equals(value, Files.readAllBytes(segment)), "Committed segment did not survive restart");
+        }
+        require(NodeIdentity.open(root, hostId).nodeId().equals(nodeId),
+            "Node identity changed after restart");
+        try {
+            new ClusterNode(root, token, repairToken, UUID.randomUUID());
+            throw new AssertionError("Node volume accepted a changed host identity");
+        } catch (IOException expected) { }
+        HttpServer oversized = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        oversized.createContext("/identity", exchange -> {
+            byte[] bytes = new byte[1024];
+            exchange.sendResponseHeaders(200, 0);
+            exchange.getResponseBody().write(bytes);
+            exchange.close();
+        });
+        oversized.createContext("/segments/", exchange -> {
+            byte[] bytes = new byte[1024];
+            exchange.sendResponseHeaders(200, 0);
+            exchange.getResponseBody().write(bytes);
+            exchange.close();
+        });
+        oversized.start();
+        try {
+            URI uri = URI.create("http://127.0.0.1:" + oversized.getAddress().getPort());
+            try {
+                NodeClient.probe(uri, token);
+                throw new AssertionError("Oversized identity response was accepted");
+            } catch (IOException expected) { }
+            NodeClient client = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, uri)), token, null);
+            try {
+                client.get(0, id, value.length, SigV4.hash(value));
+                throw new AssertionError("Oversized segment response was accepted");
+            } catch (IOException expected) { }
+        } finally { oversized.stop(0); }
+        System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup");
+    }
+    private static void require(boolean condition, String message) {
+        if (!condition) throw new AssertionError(message);
+    }
+}

test/cloud/lunarsky/store/ConcurrencyTest.java

@@ -0,0 +1,88 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public final class ConcurrencyTest {
+    private static ObjectStorage.Metadata put(DiskStore store, byte[] body) throws Exception {
+        return store.put("test", "shared", new ByteArrayInputStream(body), body.length,
+            SigV4.hex(SigV4.hash(body)), null, false, "application/octet-stream");
+    }
+
+    private static void check(DiskStore store, byte[] first, byte[] second) throws Exception {
+        try (var object = store.open("test", "shared")) {
+            byte[] body = object.stream().readAllBytes();
+            if (!Arrays.equals(body, first) && !Arrays.equals(body, second))
+                throw new AssertionError("Reader observed a partial object");
+            if (object.metadata().length() != body.length ||
+                !Arrays.equals(object.metadata().sha256(), SigV4.hash(body)))
+                throw new AssertionError("Reader observed mismatched metadata");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Path root = Files.createTempDirectory("store-concurrency-test-");
+        byte[] first = new byte[65536];
+        byte[] second = new byte[81920];
+        Arrays.fill(first, (byte) 0x35);
+        Arrays.fill(second, (byte) 0x67);
+        try (var store = new DiskStore(root, 131072, 131072);
+             var workers = Executors.newFixedThreadPool(4)) {
+            put(store, first);
+            var start = new CountDownLatch(1);
+            var done = new AtomicBoolean(false);
+            Future<?> writer = workers.submit(() -> {
+                try {
+                    start.await();
+                    for (int i = 0; i < 100; i++) {
+                        byte[] body = i % 2 == 0 ? second : first;
+                        var saved = put(store, body);
+                        var listed = store.list("test", "", "", 10, null).objects();
+                        if (listed.size() != 1 || !listed.getFirst().metadata().etag().equals(saved.etag()))
+                            throw new AssertionError("Acknowledged write is absent from listing");
+                        check(store, first, second);
+                    }
+                } catch (Exception error) {
+                    throw new RuntimeException(error);
+                } finally {
+                    done.set(true);
+                }
+            });
+            Future<?>[] readers = new Future<?>[3];
+            for (int i = 0; i < readers.length; i++) {
+                readers[i] = workers.submit(() -> {
+                    try {
+                        start.await();
+                        while (!done.get()) check(store, first, second);
+                    } catch (Exception error) {
+                        throw new RuntimeException(error);
+                    }
+                });
+            }
+            start.countDown();
+            writer.get();
+            for (Future<?> reader : readers) reader.get();
+            store.delete("test", "shared");
+            if (!store.list("test", "", "", 10, null).objects().isEmpty())
+                throw new AssertionError("Acknowledged delete is present in listing");
+            try {
+                store.open("test", "shared");
+                throw new AssertionError("Acknowledged delete remained readable");
+            } catch (StoreException expected) {
+                if (expected.status != 404) throw expected;
+            }
+            System.out.println("Concurrent storage tests passed: atomic overwrite, read and listing after acknowledgement, delete");
+        } finally {
+            try (var paths = Files.walk(root)) {
+                for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path);
+            }
+        }
+    }
+}

test/cloud/lunarsky/store/HttpTest.java

@@ -14,6 +14,8 @@ import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.concurrent.Executors;
 
 public final class HttpTest {
@@ -24,26 +26,34 @@ public final class HttpTest {
         DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);
 
     private static HttpRequest signed(String base, String method, String key, byte[] body) {
-        URI uri = URI.create(base + "/objects/" + SigV4.encode(key, true));
+        return signedUri(URI.create(base + "/objects/" + SigV4.encode(key, true)), method, body, Map.of());
+    }
+
+    private static HttpRequest signedUri(URI uri, String method, byte[] body, Map<String, String> extra) {
         String host = uri.getAuthority();
         String date = DATE.format(Instant.now());
         String hash = SigV4.hex(SigV4.hash(body));
-        String names = "host;x-amz-content-sha256;x-amz-date";
-        String canonical = method + "\n" + uri.getRawPath() + "\n\n"
-            + "host:" + host + "\n"
-            + "x-amz-content-sha256:" + hash + "\n"
-            + "x-amz-date:" + date + "\n\n" + names + "\n" + hash;
+        TreeMap<String, String> signed = new TreeMap<>(extra);
+        signed.put("host", host);
+        signed.put("x-amz-content-sha256", hash);
+        signed.put("x-amz-date", date);
+        String names = String.join(";", signed.keySet());
+        StringBuilder canonical = new StringBuilder(method).append('\n').append(uri.getRawPath()).append('\n')
+            .append(SigV4.canonicalQuery(uri.getRawQuery())).append('\n');
+        signed.forEach((name, value) -> canonical.append(name).append(':').append(value).append('\n'));
+        canonical.append('\n').append(names).append('\n').append(hash);
         String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
         String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
-            + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)));
+            + SigV4.hex(SigV4.hash(canonical.toString().getBytes(StandardCharsets.UTF_8)));
         String signature = SigV4.hex(SigV4.hmac(
             SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign));
-        return HttpRequest.newBuilder(uri)
+        HttpRequest.Builder request = HttpRequest.newBuilder(uri)
             .header("x-amz-date", date)
             .header("x-amz-content-sha256", hash)
             .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/"
-                + scope + ",SignedHeaders=" + names + ",Signature=" + signature)
-            .method(method, body.length == 0
+                + scope + ",SignedHeaders=" + names + ",Signature=" + signature);
+        extra.forEach(request::header);
+        return request.method(method, body.length == 0
                 ? HttpRequest.BodyPublishers.noBody()
                 : HttpRequest.BodyPublishers.ofByteArray(body))
             .build();
@@ -77,12 +87,85 @@ public final class HttpTest {
                 .GET().build(), HttpResponse.BodyHandlers.ofByteArray()));
             status(200, client.send(signed(base, "PUT", key, body),
                 HttpResponse.BodyHandlers.ofByteArray()));
+            String other = "folder/stars.txt";
+            status(200, client.send(signedUri(URI.create(base + "/objects/" + other), "PUT",
+                "stars".getBytes(StandardCharsets.UTF_8), Map.of("content-type", "text/plain")),
+                HttpResponse.BodyHandlers.ofByteArray()));
             var get = client.send(signed(base, "GET", key, new byte[0]),
                 HttpResponse.BodyHandlers.ofByteArray());
             status(200, get);
             if (!java.util.Arrays.equals(body, get.body())) throw new AssertionError("GET body mismatch");
             if (!"application/octet-stream".equals(get.headers().firstValue("content-type").orElse("")))
                 throw new AssertionError("Unexpected content type");
+            var typed = client.send(signed(base, "GET", other, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
+            status(200, typed);
+            if (!"text/plain".equals(typed.headers().firstValue("content-type").orElse("")))
+                throw new AssertionError("Stored content type missing");
+            var partial = client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
+                new byte[0], Map.of("range", "bytes=1-3")), HttpResponse.BodyHandlers.ofByteArray());
+            status(206, partial);
+            if (!"tar".equals(new String(partial.body(), StandardCharsets.UTF_8)) ||
+                !"bytes 1-3/5".equals(partial.headers().firstValue("content-range").orElse("")))
+                throw new AssertionError("Range response mismatch");
+            status(416, client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
+                new byte[0], Map.of("range", "bytes=20-30")), HttpResponse.BodyHandlers.ofByteArray()));
+            var listed = client.send(signedUri(URI.create(base + "/objects?list-type=2&prefix=folder%2F"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (listed.statusCode() != 200 || !listed.body().contains("<Key>folder/stars.txt</Key>") ||
+                !listed.body().contains("<Key>folder/moon-☾.txt</Key>"))
+                throw new AssertionError("ListObjectsV2 failed: " + listed.body());
+            var page = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=1"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (page.statusCode() != 200 || !page.body().contains("<IsTruncated>true</IsTruncated>"))
+                throw new AssertionError("List pagination failed: " + page.body());
+            String token = page.body().split("<NextContinuationToken>")[1].split("</NextContinuationToken>")[0];
+            var next = client.send(signedUri(URI.create(base + "/objects?list-type=2&continuation-token=" + token),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (next.statusCode() != 200 || !next.body().contains("<Key>folder/stars.txt</Key>"))
+                throw new AssertionError("List continuation failed: " + next.body());
+            var grouped = client.send(signedUri(URI.create(base + "/objects?list-type=2&delimiter=%2F"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (grouped.statusCode() != 200 || !grouped.body().contains("<CommonPrefixes><Prefix>folder/</Prefix></CommonPrefixes>") ||
+                grouped.body().contains("<Contents>"))
+                throw new AssertionError("Delimiter listing failed: " + grouped.body());
+            var encoded = client.send(signedUri(URI.create(base + "/objects?list-type=2&encoding-type=url"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (encoded.statusCode() != 200 || !encoded.body().contains("folder%2Fmoon-%E2%98%BE.txt"))
+                throw new AssertionError("Encoded listing failed: " + encoded.body());
+            var emptyPage = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=0"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (emptyPage.statusCode() != 200 || !emptyPage.body().contains("<KeyCount>0</KeyCount>"))
+                throw new AssertionError("Empty list page failed: " + emptyPage.body());
+            String movie = "folder/video.mp4";
+            URI initiate = URI.create(base + "/objects/" + movie + "?uploads=");
+            var created = client.send(signedUri(initiate, "POST", new byte[0],
+                Map.of("content-type", "video/mp4")), HttpResponse.BodyHandlers.ofString());
+            if (created.statusCode() != 200) throw new AssertionError("Multipart initiation failed: " + created.body());
+            String upload = created.body().split("<UploadId>")[1].split("</UploadId>")[0];
+            byte[] first = "hello ".getBytes(StandardCharsets.UTF_8);
+            byte[] second = "world".getBytes(StandardCharsets.UTF_8);
+            var partOne = client.send(signedUri(URI.create(base + "/objects/" + movie +
+                "?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+            var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie +
+                "?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+            status(200, partOne); status(200, partTwo);
+            String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
+                partOne.headers().firstValue("etag").orElseThrow() +
+                "</ETag></Part><Part><PartNumber>2</PartNumber><ETag>" +
+                partTwo.headers().firstValue("etag").orElseThrow() +
+                "</ETag></Part></CompleteMultipartUpload>";
+            status(200, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload),
+                "POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+            var assembled = client.send(signed(base, "GET", movie, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
+            status(200, assembled);
+            if (!"hello world".equals(new String(assembled.body(), StandardCharsets.UTF_8)) ||
+                !"video/mp4".equals(assembled.headers().firstValue("content-type").orElse("")))
+                throw new AssertionError("Completed multipart object mismatch");
+            var abandoned = client.send(signedUri(URI.create(base + "/objects/abandoned?uploads="),
+                "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            String abandonedId = abandoned.body().split("<UploadId>")[1].split("</UploadId>")[0];
+            status(204, client.send(signedUri(URI.create(base + "/objects/abandoned?uploadId=" + abandonedId),
+                "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
             var head = client.send(signed(base, "HEAD", key, new byte[0]),
                 HttpResponse.BodyHandlers.ofByteArray());
             status(200, head);
@@ -91,7 +174,7 @@ public final class HttpTest {
                 HttpResponse.BodyHandlers.ofByteArray()));
             status(404, client.send(signed(base, "GET", key, new byte[0]),
                 HttpResponse.BodyHandlers.ofByteArray()));
-            System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE");
+            System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE, MIME, ranges, listing, multipart");
         } finally {
             server.stop(0);
             executor.close();

test/cloud/lunarsky/store/StoreTest.java

@@ -2,14 +2,17 @@ package cloud.lunarsky.store;
 import java.nio.file.*;
 import java.io.*;
 import java.util.Arrays;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.util.List;
 
 public final class StoreTest {
     interface Operation {void run() throws Exception;}
     static void fails(int status,Operation operation)throws Exception{
         try{operation.run();throw new AssertionError("Expected "+status);}catch(StoreException error){if(error.status!=status)throw error;}
     }
-    static DiskStore.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
-        return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only);
+    static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
+        return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream");
     }
     public static void main(String[] args)throws Exception{
         var headers=new com.sun.net.httpserver.Headers();
@@ -35,11 +38,13 @@ public final class StoreTest {
                 fails(412,()->put(store,"../nested/☾",new byte[]{9},true));
                 fails(507,()->put(store,"second",body,true));
                 fails(413,()->put(store,"large",new byte[9],true));
-                fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true));
-                fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true));
+                fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true,"application/octet-stream"));
+                fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true,"application/octet-stream"));
                 fails(404,()->store.open("test","bad"));
                 put(store,"../nested/☾",new byte[]{9},false);
                 put(store,"empty",new byte[0],true);
+                if(store.list("test","","",100,null).objects().size()!=2)throw new AssertionError("List index");
+                if(store.objectCount()!=2||store.legacyObjects()!=0)throw new AssertionError("Object counts");
                 try {
                     new DiskStore(root,8,10);
                     throw new AssertionError("A second process opened the same data directory");
@@ -49,9 +54,53 @@ public final class StoreTest {
             }
             try(var restarted=new DiskStore(root,8,10)){
                 try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
+                if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
                 restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
                 fails(404,()->restarted.open("test","../nested/☾"));
+                var uploads=new MultipartStore(restarted);
+                String upload=uploads.create("test","from-parts","text/plain");
+                byte[] part={1,2,3};
+                uploads.putPart(upload,"test","from-parts",1,new ByteArrayInputStream(part),part.length,
+                    SigV4.hex(SigV4.hash(part)),null);
+                Files.writeString(root.resolve("pending-upload-id"),upload);
             }
+            try(var resumed=new DiskStore(root,8,10)){
+                Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000");
+                Files.createDirectory(unfinished);
+                Files.write(unfinished.resolve("manifest"),new byte[]{1,2,3});
+                var uploads=new MultipartStore(resumed);
+                if(Files.exists(unfinished))throw new AssertionError("Unfinished multipart creation survived restart");
+                String upload=Files.readString(root.resolve("pending-upload-id"));
+                byte[] part={1,2,3};
+                String etag=SigV4.hex(MessageDigest.getInstance("MD5").digest(part));
+                uploads.complete(upload,"test","from-parts",List.of(new MultipartStorage.Part(1,etag)));
+                try(var obj=resumed.open("test","from-parts")){
+                    if(!Arrays.equals(part,obj.stream().readAllBytes())||!obj.metadata().contentType().equals("text/plain"))
+                        throw new AssertionError("Multipart restart");
+                }
+                resumed.delete("test","from-parts");
+            }
+            byte[] old={4,5,6};
+            String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
+            Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId);
+            Files.createDirectories(oldPath.getParent());
+            ByteBuffer oldRecord=ByteBuffer.allocate(72+old.length).putLong(0x4c534f424a303031L)
+                .putLong(old.length).putLong(123456789L)
+                .put(MessageDigest.getInstance("MD5").digest(old)).put(SigV4.hash(old)).put(old);
+            Files.write(oldPath,oldRecord.array());
+            try(var migrated=new DiskStore(root,8,10)){
+                if(migrated.objectCount()!=2||migrated.legacyObjects()!=1)throw new AssertionError("Legacy counts");
+                try(var obj=migrated.open("test","legacy")){
+                    if(!Arrays.equals(old,obj.stream().readAllBytes()))throw new AssertionError("Legacy read");
+                }
+                if(migrated.list("test","","",100,null).objects().stream().anyMatch(entry->entry.key().equals("legacy")))
+                    throw new AssertionError("Legacy object appeared without a stored key");
+                put(migrated,"legacy",old,false);
+                if(migrated.objectCount()!=2||migrated.legacyObjects()!=0)throw new AssertionError("Legacy count after overwrite");
+                if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy")))
+                    throw new AssertionError("Legacy overwrite was not indexed");
+            }
+            Files.delete(root.resolve("pending-upload-id"));
             String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
             Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
             try {
@@ -61,7 +110,7 @@ public final class StoreTest {
                 if(!expected.getMessage().contains("object record"))throw expected;
             }
             try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
-            System.out.println("Java storage tests passed: roundtrip, quota, integrity, persistence, locking, corruption, delete");
+            System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete");
         }finally{try(var paths=Files.walk(root)){for(var p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}}
     }
 }