Release ObjectStore 0.0.2

.env.cluster.example

@@ -0,0 +1,7 @@
+S3_ACCESS_KEY=
+S3_SECRET_KEY=
+CLUSTER_TOKEN=
+CLUSTER_REPAIR_TOKEN=
+POSTGRES_PASSWORD=
+S3_BUCKET=objects
+CLUSTER_HOST_PORT=9001

.env.example

@@ -5,5 +5,5 @@ S3_SECRET_KEY=
 S3_BUCKET=objects
 S3_REGION=us-east-1
 HOST_PORT=9000
-MAX_OBJECT_BYTES=10485760
+MAX_OBJECT_BYTES=134217728
 MAX_TOTAL_BYTES=2147483648

.gitignore

@@ -1,4 +1,10 @@
 out/
 .env
+.env.cluster
+*.local.env
 data/
 *.log
+__pycache__/
+*.pyc
+CLUSTER_PLAN.md
+SCALE_OUT_PLAN.md

Dockerfile

@@ -1,14 +1,22 @@
 FROM eclipse-temurin:21-jdk-alpine AS build
 WORKDIR /src
+RUN wget -q -O /tmp/postgresql.jar https://jdbc.postgresql.org/download/postgresql-42.7.14.jar && \
+    echo '73914527305a40cce504b0d3d90b23caf565136912d607ae8ae7c5895512332c  /tmp/postgresql.jar' | sha256sum -c -
 COPY src ./src
 COPY test ./test
 RUN mkdir /out && javac --release 21 --add-modules jdk.httpserver,java.net.http -d /out src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java
 RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.StoreTest
+RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.ConcurrencyTest
 RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.HttpTest
+RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.ClusterNodeTest
+RUN java -cp /out cloud.lunarsky.store.CliTest
 
 FROM eclipse-temurin:21-jre-alpine
 RUN addgroup -g 10001 store && adduser -D -u 10001 -G store store && mkdir /data && chown store:store /data
 COPY --from=build /out /app
+COPY --from=build /tmp/postgresql.jar /app/postgresql.jar
+COPY scripts/objectstore /usr/local/bin/objectstore
+RUN chmod 755 /usr/local/bin/objectstore
 USER store
 EXPOSE 9000
-ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver", "-cp", "/app", "cloud.lunarsky.store.Main"]
+ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.Main"]

README.md

@@ -1,6 +1,58 @@
+![LunarSky ObjectStore banner](assets/objectstore-banner.svg)
+
 # ObjectStore
 
-Small S3-compatible object storage for LunarSky. Early development.
+Small S3-compatible object storage for LunarSky.
+
+Source: [GitHub](https://github.com/LunarSkyOSS/ObjectStore) · [Gitea mirror](https://git.lunarsky.cloud/admin/ObjectStore)
+
+**Development software. Do not use it for production data.** It is provided as is, without warranty, under the [MIT License](LICENSE).
+
+[![Stage: development](assets/badge-stage.svg)](#limits-and-safety) [![License: MIT](assets/badge-license.svg)](LICENSE) [![Runtime: Java 21](assets/badge-java.svg)](Dockerfile) [![S3 API: partial support](assets/badge-api.svg)](#s3-api-support-checklist)
+
+## Contents
+
+- [Capability checklist](#capability-checklist)
+- [S3 API support checklist](#s3-api-support-checklist)
+- [Tests](TESTS.md)
+- [Single-node setup](#single-node-setup)
+- [CLI and tests](#cli-and-tests)
+- [Local cluster prototype](#local-cluster-prototype)
+- [Migrating a local cluster](#migrating-a-local-cluster)
+- [Adding a cluster node](#adding-a-cluster-node)
+- [Limits and safety](#limits-and-safety)
+- [Disclaimer](#disclaimer)
+- [AI contributions](#ai-contributions)
+
+## Capability checklist
+
+ObjectStore serves one configured bucket.
+
+- ✅ Persistent single-node storage with checksum verification
+- ✅ Configurable per-object and total logical size limits
+- ✅ CLI status, version, and full payload verification
+- ✅ Local cluster prototype with stable node IDs and host-aware placement code
+- ⬜ Automatic repair, rebalance, and garbage collection
+- ⬜ Production multi-server deployment and metadata failover
+
+New objects retain their content type and key. Objects written by the earlier single-node format remain readable, but cannot appear in listings until overwritten because their original keys were not stored.
+
+## S3 API support checklist
+
+- ✅ Header-based AWS Signature Version 4 authentication
+- ✅ `PutObject`, `GetObject`, `HeadObject`, and `DeleteObject` in both modes
+- ✅ Single-range GET and `ListObjectsV2` in both modes
+- ✅ SHA-256 payload verification and `x-amz-checksum-sha256` in both modes
+- ✅ `CreateMultipartUpload`, `UploadPart`, `CompleteMultipartUpload`, and `AbortMultipartUpload` in single-node mode
+- ⬜ Multipart uploads in cluster mode
+- ⬜ Presigned URLs and streaming Signature V4 uploads
+- ⬜ `CopyObject`, `ListParts`, and `ListMultipartUploads`
+- ⬜ `Content-MD5` and checksum algorithms other than SHA-256
+- ⬜ Bucket creation and listing, object versioning, ACLs, tags, and user metadata
+
+This is an S3 API subset, not full AWS S3 compatibility. Unsupported S3 operations and Amazon-specific headers are rejected.
+
+## Single-node setup
 
 Requires Docker Compose. Copy `.env.example` to `.env`, then set `S3_ACCESS_KEY` and `S3_SECRET_KEY` to unique values. The access key must be at least 16 alphanumeric characters; the secret must be at least 32 characters.
 
@@ -12,6 +64,64 @@ curl http://127.0.0.1:9000/health
 
 Port 9000 binds to localhost. Data stays in the `object-data` Docker volume. `docker compose down -v` deletes that volume.
 
-Run `sh scripts/test.sh` with JDK 21 to test from source. Lunaris uses ObjectStore through its S3 storage adapter.
+The standalone defaults are 128 MiB per object and 2 GiB total. Set `MAX_OBJECT_BYTES` and `MAX_TOTAL_BYTES` in `.env` to change them. Incomplete multipart uploads consume space until aborted.
 
-Do not use this development version as the only copy of important data.
+## CLI and tests
+
+From the server shell, run the CLI inside the running container from the directory containing `compose.yaml`:
+
+```sh
+docker compose exec objectstore objectstore status
+docker compose exec objectstore objectstore verify
+docker compose exec objectstore objectstore version
+```
+
+The startup log shows the LunarSky banner, version, and a small storage summary (`docker compose logs --tail=20 objectstore`). `status` reports object and multipart usage. `verify` also checks stored payload hashes and exits nonzero on an error. Both commands can run while the service is live; they are not a snapshot or a backup.
+
+Run `sh scripts/test.sh` with JDK 21 to test from source. See [TESTS.md](TESTS.md) for coverage, the disposable Docker cluster suite, and the limits of those tests.
+
+## Local cluster prototype
+
+The local cluster prototype starts three segment containers and one PostgreSQL container on the same Docker host. Copy `.env.cluster.example` to a private environment file, replace all four credentials, and run:
+
+```sh
+docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml up -d --build
+sh scripts/test-cluster.sh /path/to/cluster.env
+docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml run --rm repair
+```
+
+The cluster S3 endpoint binds to `127.0.0.1:9001`; storage nodes and PostgreSQL have no published ports. The separate repair container holds the repair credential and restores missing or corrupt replicas.
+
+Node UUIDs persist on their volumes, and replica manifests use those UUIDs so reordering configured URLs cannot move an existing replica. Each node also has an operator-assigned physical host UUID. New writes require acknowledgements from two different host UUIDs. The optional `CLUSTER_TEST_NODE_DOMAINS=true` override counts containers instead, solely for local process tests; all containers in this Compose file share one physical host.
+
+## Migrating a local cluster
+
+For an existing **local** three-node cluster that stores replicas by URL position:
+
+1. Stop the old gateway. Back up PostgreSQL and every node volume, and record the original ordered `CLUSTER_NODES`.
+2. Start the nodes with the new image and a separate `CLUSTER_REPAIR_TOKEN`. Do not start the new gateway yet.
+3. Run `objectstore cluster-migrate --check` in a one-off gateway container. Compare `legacy_0` through `legacy_2` and their URLs with the pre-upgrade inventory.
+4. Run `objectstore cluster-migrate --apply id0,id1,id2` with those node IDs in the original order. The command verifies every listed live replica before conversion.
+5. Start the new gateway only after the command reports `cluster_format=2`. Do not restart an old gateway against the converted database.
+
+The old format did not record the original URL mapping, so the inventory check is essential. The isolated migration fixture tests conversion; building this code does not automatically upgrade the running local prototype.
+
+## Adding a cluster node
+
+`objectstore cluster-join http://new-node:9100 expected-host-uuid` registers an additional local node. Add its URL to `CLUSTER_NODES` and restart the gateway to use it for new writes. Existing segments stay where their manifests say; this is capacity expansion for new writes, not a rebalance. `scripts/test-cluster.sh` exercises a fourth container joining and receiving new segments.
+
+## Limits and safety
+
+The cluster retains old and failed-write segments. It has no garbage collection, metadata standby, automated rebalance, private-network TLS, scoped credentials, or physical host verification yet. Host UUIDs are operator labels, not proof that machines have separate power, disks, or network paths. Keep `CLUSTER_LOCAL_DEV=true` limited to local tests.
+
+The standalone cluster node binds to localhost by default. Set `NODE_BIND` only for a private test network; the Compose file binds inside its private Docker network. PostgreSQL JDBC 42.7.14 is bundled in the image with its license inside the JAR.
+
+## Disclaimer
+
+ObjectStore is independent software. It is not affiliated with, sponsored by, or endorsed by Amazon or Amazon Web Services (AWS). “S3-compatible” describes only the API subset listed above.
+
+Keep independent backups; data loss is possible. The software is provided “as is” under the [MIT License](LICENSE). To the extent permitted by applicable law, the authors and maintainers are not responsible for data loss or other damages arising from its use.
+
+## AI contributions
+
+AI tools assisted with parts of this project. Maintainers review releases and remain responsible for what ships.

TESTS.md

@@ -0,0 +1,60 @@
+# Testing ObjectStore
+
+ObjectStore is development software. Do not use it for production data. It is provided **as is, without warranty** under the [MIT License](LICENSE). Keep independent backups. Passing these tests does not guarantee that data cannot be lost.
+
+Run the commands below from the repository root.
+
+## Source tests
+
+Requires JDK 21. No Docker service is needed.
+
+```sh
+sh scripts/test.sh
+```
+
+The script compiles the source and test programs into `out/classes`, then runs:
+
+| Test | Checks |
+| --- | --- |
+| `StoreTest` | Signature V4 test vector and tampering, local writes and reads, quotas, restart persistence, multipart recovery, legacy reads, locking, and corruption rejection. |
+| `ConcurrencyTest` | Atomic local overwrites and consistent reads, listings, and deletes during concurrent access. |
+| `HttpTest` | Signed HTTP requests, object operations, ranges, listing, and single-node multipart uploads. |
+| `ClusterNodeTest` | Node identity and locking, authenticated segment transfers, checksum rejection, repair authorization, and restart cleanup. |
+| `CliTest` | Version, status, verification, and a nonzero result for corrupt data. |
+
+The script exits nonzero on failure. The test programs use temporary local directories and loopback HTTP ports; they do not use an existing ObjectStore volume.
+
+## Disposable Docker cluster tests
+
+Requires Docker with Compose, Python 3, `curl`, and a free local port 9001. Make a test-only environment file from `.env.cluster.example` and fill in all five blank credentials with test-only values. Keep that file private and out of Git.
+
+```sh
+cp .env.cluster.example /tmp/objectstore-cluster-tests.env
+chmod 600 /tmp/objectstore-cluster-tests.env
+```
+
+After filling in the file, run:
+
+```sh
+COMPOSE_PROJECT_NAME=objectstore-tests sh scripts/test-cluster.sh /tmp/objectstore-cluster-tests.env
+```
+
+Use a fresh, disposable Compose project. The script writes test objects, stops and restarts storage nodes and PostgreSQL, corrupts a replica to exercise repair, and joins a fourth node. It leaves the test stack running. To remove **only that test project's** containers and volumes after review:
+
+```sh
+COMPOSE_PROJECT_NAME=objectstore-tests docker compose --env-file /tmp/objectstore-cluster-tests.env -f compose.cluster.yaml --profile expansion down -v
+```
+
+If port 9001 is occupied, set `CLUSTER_HOST_PORT` to the same free port in both the environment file and the shell before running the script. The script reads that port from the shell; Compose reads it from the file.
+
+The Docker suite checks signed S3 operations, multi-segment objects, concurrent overwrites, reads and writes with a node stopped, refusal to write without a storage quorum, restart recovery, corrupt-replica repair, metadata unavailability, and placement on a newly joined node. It also checks that containers labeled as one physical host cannot satisfy the normal host quorum. Its local-only override permits the remaining phases to use containers as separate test domains.
+
+`ClusterMigrationTest` is a separate legacy-format fixture and is **not** run by either test script. Do not run its `create` phase against a populated metadata database. The migration procedure is in the [README](README.md#migrating-a-local-cluster).
+
+## What these tests do not prove
+
+- Container stops are not physical power cuts or disk failures. The automated suite does not reboot a host or test every possible crash point.
+- The Compose nodes share one machine. Passing the local-only quorum override does not demonstrate durability across independent hosts, racks, or sites.
+- The suite does not test metadata failover, an off-site backup restore, prolonged load, or full AWS S3 compatibility.
+
+See [Limits and safety](README.md#limits-and-safety) before evaluating any multi-server deployment.

assets/badge-api.svg

@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="130" height="28" viewBox="0 0 130 28" role="img" aria-label="S3 API: partial support">
+  <clipPath id="edge"><rect width="130" height="28" rx="6"/></clipPath>
+  <g clip-path="url(#edge)">
+    <rect width="130" height="28" fill="#f1e8f8"/>
+    <rect width="62" height="28" fill="#25304e"/>
+  </g>
+  <text x="12" y="18.5" fill="#ffffff" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="700" letter-spacing=".4">S3 API</text>
+  <text x="74" y="18.5" fill="#5b3369" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="700">PARTIAL</text>
+</svg>

assets/badge-java.svg

@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="139" height="28" viewBox="0 0 139 28" role="img" aria-label="Runtime: Java 21">
+  <clipPath id="edge"><rect width="139" height="28" rx="6"/></clipPath>
+  <g clip-path="url(#edge)">
+    <rect width="139" height="28" fill="#e4efff"/>
+    <rect width="74" height="28" fill="#25304e"/>
+  </g>
+  <text x="10" y="18.5" fill="#ffffff" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="700" letter-spacing=".4">RUNTIME</text>
+  <text x="86" y="18.5" fill="#25486a" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="700">JAVA 21</text>
+</svg>

assets/badge-license.svg

@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="126" height="28" viewBox="0 0 126 28" role="img" aria-label="License: MIT">
+  <clipPath id="edge"><rect width="126" height="28" rx="6"/></clipPath>
+  <g clip-path="url(#edge)">
+    <rect width="126" height="28" fill="#e0f5ed"/>
+    <rect width="75" height="28" fill="#25304e"/>
+  </g>
+  <text x="11" y="18.5" fill="#ffffff" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="700" letter-spacing=".4">LICENSE</text>
+  <text x="88" y="18.5" fill="#214b43" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="700">MIT</text>
+</svg>

assets/badge-stage.svg

@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="176" height="28" viewBox="0 0 176 28" role="img" aria-label="Stage: development">
+  <clipPath id="edge"><rect width="176" height="28" rx="6"/></clipPath>
+  <g clip-path="url(#edge)">
+    <rect width="176" height="28" fill="#eceef1"/>
+    <rect width="61" height="28" fill="#3d4048"/>
+  </g>
+  <text x="12" y="18.5" fill="#ffffff" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="700" letter-spacing=".4">STAGE</text>
+  <text x="72" y="18.5" fill="#30343c" font-family="Inter, Arial, sans-serif" font-size="11" font-weight="650">DEVELOPMENT</text>
+</svg>

assets/objectstore-banner.svg

@@ -0,0 +1,61 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1600 300" role="img" aria-labelledby="title desc">
+  <title id="title">LunarSky ObjectStore</title>
+  <desc id="desc">LunarSky woven hexagon logo and ObjectStore title on white with a soft prismatic gradient.</desc>
+  <defs>
+    <linearGradient id="paper" x2="0" y2="1">
+      <stop stop-color="#ffffff"/>
+      <stop offset="1" stop-color="#f9fbff"/>
+    </linearGradient>
+    <linearGradient id="logo-gradient" x1="0" y1="0" x2="1" y2="1">
+      <stop stop-color="#63ddcb"/>
+      <stop offset=".5" stop-color="#75a7ed"/>
+      <stop offset="1" stop-color="#aa78ea"/>
+    </linearGradient>
+    <linearGradient id="ribbon" x1="0" y1="1" x2="1" y2="0">
+      <stop stop-color="#eff9fd"/>
+      <stop offset=".24" stop-color="#caefe8"/>
+      <stop offset=".55" stop-color="#b9c7fa"/>
+      <stop offset=".78" stop-color="#d7b5f4"/>
+      <stop offset="1" stop-color="#f5cadc"/>
+    </linearGradient>
+    <linearGradient id="edge" x1="0" y1="1" x2="1" y2="0">
+      <stop stop-color="#73dfd1" stop-opacity="0"/>
+      <stop offset=".4" stop-color="#8cbbfa" stop-opacity=".55"/>
+      <stop offset=".75" stop-color="#ae94ef" stop-opacity=".78"/>
+      <stop offset="1" stop-color="#e6a8d3" stop-opacity="0"/>
+    </linearGradient>
+    <radialGradient id="glow">
+      <stop stop-color="#d5e0ff" stop-opacity=".72"/>
+      <stop offset=".65" stop-color="#e8d7f9" stop-opacity=".36"/>
+      <stop offset="1" stop-color="#ffffff" stop-opacity="0"/>
+    </radialGradient>
+    <filter id="soften" x="-40%" y="-80%" width="180%" height="260%">
+      <feGaussianBlur stdDeviation="31"/>
+    </filter>
+    <mask id="weave">
+      <g fill="#fff">
+        <path id="blade" d="M64 13 96 31 87 46 64 33 64 49 48 40 48 22Z"/>
+        <use href="#blade" transform="rotate(60 64 64)"/>
+        <use href="#blade" transform="rotate(120 64 64)"/>
+        <use href="#blade" transform="rotate(180 64 64)"/>
+        <use href="#blade" transform="rotate(240 64 64)"/>
+        <use href="#blade" transform="rotate(300 64 64)"/>
+      </g>
+    </mask>
+  </defs>
+
+  <rect width="1600" height="300" fill="url(#paper)"/>
+  <ellipse cx="1335" cy="114" rx="365" ry="226" fill="url(#glow)"/>
+  <path d="M986 300C1129 232 1213 134 1308 86c85-43 174-45 292-61v275Z" fill="url(#ribbon)" opacity=".77"/>
+  <path d="M1023 311c150-89 243-225 381-253 69-14 132-12 217-34" fill="none" stroke="url(#edge)" stroke-width="74" opacity=".45" filter="url(#soften)"/>
+  <path d="M1024 300c155-91 253-204 398-235 68-14 119-12 178-17" fill="none" stroke="url(#edge)" stroke-width="2" opacity=".7"/>
+  <path d="M1134 300c121-60 222-154 340-172 44-7 85-7 126-7" fill="none" stroke="#ffffff" stroke-width="1.5" opacity=".64"/>
+  <path d="M1250 300c97-39 181-99 276-105 24-2 49-2 74-1" fill="none" stroke="#ffffff" stroke-width="1.5" opacity=".5"/>
+
+  <g transform="translate(62 86)">
+    <rect width="128" height="128" fill="url(#logo-gradient)" mask="url(#weave)"/>
+  </g>
+  <text x="224" y="171" fill="#202b45" font-family="Inter, Arial, sans-serif" font-size="52" font-weight="650" letter-spacing="-2">LunarSky</text>
+  <path d="M522 99v103" stroke="#cbd5e6" stroke-width="2"/>
+  <text x="562" y="178" fill="#17233d" font-family="Inter Display, Inter, Arial, sans-serif" font-size="76" font-weight="700" letter-spacing="-4">ObjectStore</text>
+</svg>

compose.cluster.yaml

@@ -0,0 +1,127 @@
+services:
+  gateway:
+    build: .
+    image: lunarsky-objectstore:cluster-local
+    restart: unless-stopped
+    environment:
+      STORE_MODE: cluster
+      CLUSTER_LOCAL_DEV: "true"
+      CLUSTER_TEST_NODE_DOMAINS: "true"
+      BIND_ADDRESS: 0.0.0.0
+      S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
+      S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
+      S3_BUCKET: ${S3_BUCKET:-objects}
+      S3_REGION: ${S3_REGION:-us-east-1}
+      MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728}
+      MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648}
+      CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
+      CLUSTER_NODES: ${CLUSTER_NODES:-http://node-a:9100,http://node-b:9100,http://node-c:9100}
+      POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    ports:
+      - "127.0.0.1:${CLUSTER_HOST_PORT:-9001}:9000"
+    depends_on:
+      metadata:
+        condition: service_healthy
+      node-a:
+        condition: service_healthy
+      node-b:
+        condition: service_healthy
+      node-c:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9000/ready"]
+      interval: 10s
+      timeout: 4s
+      retries: 3
+    mem_limit: 384m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
+  metadata:
+    image: postgres:17-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: objectstore
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    volumes:
+      - cluster-metadata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U objectstore -d objectstore"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    mem_limit: 256m
+    security_opt:
+      - no-new-privileges:true
+
+  repair:
+    image: lunarsky-objectstore:cluster-local
+    profiles: [maintenance]
+    entrypoint: ["/usr/local/bin/objectstore", "cluster-repair"]
+    environment:
+      STORE_MODE: cluster
+      CLUSTER_LOCAL_DEV: "true"
+      CLUSTER_TEST_NODE_DOMAINS: "true"
+      CLUSTER_NODES: ${CLUSTER_NODES:-http://node-a:9100,http://node-b:9100,http://node-c:9100}
+      CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
+      CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN}
+      S3_BUCKET: ${S3_BUCKET:-objects}
+      POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    mem_limit: 384m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
+  node-a: &node
+    image: lunarsky-objectstore:cluster-local
+    restart: unless-stopped
+    entrypoint: ["java", "-XX:MaxRAMPercentage=70", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.ClusterNode"]
+    environment:
+      CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
+      CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN}
+      CLUSTER_HOST_ID: 5f1447b5-3f9e-457b-8ee7-e26f0c475b5f
+      DATA_DIR: /data
+      NODE_BIND: 0.0.0.0
+    volumes:
+      - cluster-node-a:/data
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/health"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+    mem_limit: 256m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
+  node-b:
+    <<: *node
+    volumes:
+      - cluster-node-b:/data
+
+  node-c:
+    <<: *node
+    volumes:
+      - cluster-node-c:/data
+
+  node-d:
+    <<: *node
+    profiles: [expansion]
+    volumes:
+      - cluster-node-d:/data
+
+volumes:
+  cluster-metadata:
+  cluster-node-a:
+  cluster-node-b:
+  cluster-node-c:
+  cluster-node-d:

compose.yaml

@@ -8,7 +8,7 @@ services:
       S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env}
       S3_BUCKET: ${S3_BUCKET:-objects}
       S3_REGION: ${S3_REGION:-us-east-1}
-      MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-10485760}
+      MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728}
       MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648}
     ports:
       - "127.0.0.1:${HOST_PORT:-9000}:9000"

scripts/objectstore

@@ -0,0 +1,14 @@
+#!/bin/sh
+if [ "${1:-}" = "cluster-repair" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterRepair "$@"
+fi
+if [ "${1:-}" = "cluster-migrate" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterMigrate "$@"
+fi
+if [ "${1:-}" = "cluster-join" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterJoin "$@"
+fi
+exec java -XX:MaxRAMPercentage=70 -cp /app cloud.lunarsky.store.Cli "$@"

scripts/test-cluster-http.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import datetime
+import hashlib
+import hmac
+import pathlib
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+values = dict(line.strip().split("=", 1) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()
+         if line.strip() and not line.startswith("#"))
+access = values["S3_ACCESS_KEY"]
+secret = values["S3_SECRET_KEY"]
+bucket = values.get("S3_BUCKET", "objects")
+port = values.get("CLUSTER_HOST_PORT", "9001")
+host = f"127.0.0.1:{port}"
+
+
+def sign(key, message):
+    return hmac.new(key, message.encode(), hashlib.sha256).digest()
+
+
+def request(method, path, body=b"", extra=None):
+    extra = extra or {}
+    date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    stamp = date[:8]
+    digest = hashlib.sha256(body).hexdigest()
+    uri, _, query = path.partition("?")
+    canonical = "&".join(f"{urllib.parse.quote(k, safe='~-._')}={urllib.parse.quote(v, safe='~-._')}"
+                         for k, v in sorted(urllib.parse.parse_qsl(query, keep_blank_values=True)))
+    headers = {"host": host, "x-amz-content-sha256": digest, "x-amz-date": date, **extra}
+    signed_names = ";".join(sorted(headers))
+    canonical_headers = "".join(f"{name}:{headers[name]}\n" for name in sorted(headers))
+    canonical_request = f"{method}\n{uri}\n{canonical}\n{canonical_headers}\n{signed_names}\n{digest}"
+    scope = f"{stamp}/us-east-1/s3/aws4_request"
+    to_sign = f"AWS4-HMAC-SHA256\n{date}\n{scope}\n{hashlib.sha256(canonical_request.encode()).hexdigest()}"
+    key = sign(sign(sign(sign(("AWS4" + secret).encode(), stamp), "us-east-1"), "s3"), "aws4_request")
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={access}/{scope},"
+                                f"SignedHeaders={signed_names},Signature={signature}")
+    url = f"http://{host}{path}"
+    outgoing = urllib.request.Request(url, data=body if method == "PUT" else None,
+                                      method=method, headers=headers)
+    try:
+        with urllib.request.urlopen(outgoing, timeout=30) as response:
+            return response.status, response.read(), response.headers
+    except urllib.error.HTTPError as error:
+        return error.code, error.read(), error.headers
+
+
+if len(sys.argv) > 2 and sys.argv[2] == "survivor":
+    status, content, _ = request("GET", f"/{bucket}/cluster-test/survivor")
+    assert status == 200 and content == b"acknowledged object survives node loss", (status, content)
+    print("Cluster surviving-replica HTTP read passed")
+    sys.exit(0)
+
+if len(sys.argv) > 4 and sys.argv[2] == "status":
+    key = urllib.parse.quote(sys.argv[3], safe="/")
+    expected = int(sys.argv[4])
+    status, _, _ = request("GET", f"/{bucket}/{key}")
+    assert status == expected, (status, expected)
+    print(f"Cluster GET status passed: {status}")
+    sys.exit(0)
+
+
+key = f"/{bucket}/cluster-test/http.txt"
+body = b"HTTP gateway integration test"
+status, _, _ = request("PUT", key, body)
+assert status == 200, status
+status, content, _ = request("GET", key)
+assert status == 200 and content == body, (status, content)
+status, content, _ = request("GET", key, extra={"range": "bytes=5-11"})
+assert status == 206 and content == body[5:12], (status, content)
+status, content, _ = request("HEAD", key)
+assert status == 200 and not content, status
+status, content, _ = request("GET", f"/{bucket}?list-type=2&prefix=cluster-test%2F")
+assert status == 200 and b"cluster-test/http.txt" in content, (status, content)
+status, _, _ = request("DELETE", key)
+assert status == 204, status
+status, _, _ = request("GET", key)
+assert status == 404, status
+print("Cluster HTTP tests passed: signed PUT, GET, range, HEAD, LIST, DELETE")

scripts/test-cluster.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+env_file=${1:?Usage: sh scripts/test-cluster.sh /path/to/local-cluster.env}
+host_port=${CLUSTER_HOST_PORT:-9001}
+compose() { docker compose --env-file "$env_file" -f compose.cluster.yaml "$@"; }
+restore() { compose start metadata node-a node-b >/dev/null 2>&1 || true; }
+trap restore EXIT
+compose up -d --build
+run_phase() {
+  compose exec -T gateway java --add-modules jdk.httpserver,java.net.http \
+    -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterIntegrationTest "$1"
+}
+wait_ready() {
+  attempt=0
+  until curl -fsS -o /dev/null "http://127.0.0.1:$host_port/ready" 2>/dev/null; do
+    attempt=$((attempt + 1))
+    [ "$attempt" -lt 30 ] || return 1
+    sleep 1
+  done
+}
+run_phase basic
+run_phase same-host
+run_phase concurrent
+compose stop node-a
+run_phase degraded
+compose stop node-b
+run_phase quorum-lost
+compose start node-a node-b
+wait_ready
+run_phase recovered
+segment_id=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT s.segment_id FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE o.object_key='cluster-test/survivor' LIMIT 1")
+printf '%s\n' "$segment_id" | grep -Eq '^[0-9a-f-]{36}$'
+shard=$(printf '%s' "$segment_id" | cut -c1-2)
+compose exec -T node-a sh -c 'printf corrupted > "/data/segments/$1/$2"' _ "$shard" "$segment_id"
+run_phase recovered
+compose run --rm -T repair
+expected=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT encode(s.sha256,'hex') FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE o.object_key='cluster-test/survivor' LIMIT 1")
+actual=$(compose exec -T node-a sha256sum "/data/segments/$shard/$segment_id" | cut -d' ' -f1)
+[ "$expected" = "$actual" ]
+compose stop metadata
+status=$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:$host_port/ready")
+[ "$status" = 503 ]
+compose start metadata
+wait_ready
+run_phase recovered
+python3 scripts/test-cluster-http.py "$env_file"
+compose --profile expansion up -d node-d
+compose run --rm -T --no-deps --entrypoint /usr/local/bin/objectstore gateway \
+  cluster-join http://node-d:9100 5f1447b5-3f9e-457b-8ee7-e26f0c475b5f
+export CLUSTER_NODES=http://node-a:9100,http://node-b:9100,http://node-c:9100,http://node-d:9100
+compose up -d --no-deps gateway
+wait_ready
+run_phase recovered
+run_phase joined
+echo 'Cluster failure tests passed'

scripts/test.sh

@@ -5,4 +5,7 @@ mkdir -p out/classes
 javac --release 21 --add-modules jdk.httpserver,java.net.http -d out/classes \
   src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java
 java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.StoreTest
+java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.ConcurrencyTest
 java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.HttpTest
+java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.ClusterNodeTest
+java -cp out/classes cloud.lunarsky.store.CliTest

src/cloud/lunarsky/store/Cli.java

@@ -0,0 +1,154 @@
+package cloud.lunarsky.store;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.List;
+
+public final class Cli {
+    private Cli() {}
+
+    static final class Report {
+        long objects, legacyObjects, payloadBytes, recordBytes, uploads, stagedBytes, checked, changedDuringScan;
+        long errors;
+        final List<String> problems = new ArrayList<>();
+
+        void problem(String message) {
+            errors++;
+            if (problems.size() < 100) problems.add(message);
+        }
+    }
+
+    public static void main(String[] args) {
+        int result = run(args, Path.of(System.getenv().getOrDefault("DATA_DIR", "/data")), System.out, System.err);
+        if (result != 0) System.exit(result);
+    }
+
+    static int run(String[] args, Path root, PrintStream out, PrintStream err) {
+        if (args.length == 1 && (args[0].equals("version") || args[0].equals("--version"))) {
+            out.println("ObjectStore " + Version.VALUE);
+            return 0;
+        }
+        if (args.length == 1 && (args[0].equals("help") || args[0].equals("--help"))) {
+            out.println("Usage: objectstore status|verify|version");
+            out.println("status  Show stored object and multipart usage");
+            out.println("verify  Check object records, paths and payload checksums");
+            out.println("version Show the ObjectStore version");
+            return 0;
+        }
+        if (args.length != 1 || !(args[0].equals("status") || args[0].equals("verify"))) {
+            err.println("Usage: objectstore status|verify|version");
+            return 2;
+        }
+        try {
+            boolean verify = args[0].equals("verify");
+            Report report = inspect(root, verify);
+            out.println("version=" + Version.VALUE);
+            out.println("objects=" + report.objects);
+            out.println("legacy_objects=" + report.legacyObjects);
+            out.println("payload_bytes=" + report.payloadBytes);
+            out.println("record_bytes=" + report.recordBytes);
+            out.println("multipart_uploads=" + report.uploads);
+            out.println("multipart_staged_bytes=" + report.stagedBytes);
+            if (verify) out.println("verified_objects=" + report.checked);
+            if (report.changedDuringScan > 0) out.println("changed_during_scan=" + report.changedDuringScan);
+            out.println("errors=" + report.errors);
+            for (String problem : report.problems) err.println(problem);
+            if (report.errors > report.problems.size())
+                err.println((report.errors - report.problems.size()) + " further errors omitted");
+            return report.errors == 0 ? 0 : 1;
+        } catch (IOException error) {
+            err.println("ObjectStore inspection failed: " + error.getMessage());
+            return 1;
+        }
+    }
+
+    static Report inspect(Path root, boolean verify) throws IOException {
+        Path objects = root.resolve("objects");
+        if (!Files.isDirectory(objects)) throw new IOException("Object data directory does not exist: " + objects);
+        Report report = new Report();
+        try (var paths = Files.walk(objects)) {
+            for (Path path : paths.filter(Files::isRegularFile).toList()) inspectObject(objects, path, verify, report);
+        }
+        Path multipart = root.resolve("multipart");
+        if (Files.isDirectory(multipart)) {
+            try (var uploads = Files.list(multipart)) {
+                for (Path dir : uploads.toList()) inspectUpload(dir, report);
+            }
+        }
+        return report;
+    }
+
+    private static void inspectObject(Path objects, Path path, boolean verify, Report report) {
+        try (var channel = FileChannel.open(path, StandardOpenOption.READ);
+             var input = new DataInputStream(Channels.newInputStream(channel))) {
+            long size = channel.size();
+            var record = DiskStore.readRecord(input);
+            var meta = record.metadata();
+            report.objects++;
+            report.payloadBytes += meta.length();
+            report.recordBytes += size;
+            if (meta.key() == null) report.legacyObjects++;
+            else {
+                String id = SigV4.hex(SigV4.hash((meta.bucket() + "/" + meta.key()).getBytes(StandardCharsets.UTF_8)));
+                Path expected = objects.resolve(id.substring(0, 2)).resolve(id);
+                if (!path.equals(expected)) report.problem("Mismatched object path: " + path);
+            }
+            if (size - record.headerLength() != meta.length()) {
+                report.problem("Invalid object length: " + path);
+                return;
+            }
+            if (verify) {
+                MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
+                byte[] buffer = new byte[65536]; long count = 0; int n;
+                while ((n = input.read(buffer)) != -1) {
+                    count += n;
+                    sha.update(buffer, 0, n);
+                    md5.update(buffer, 0, n);
+                }
+                report.checked++;
+                if (count != meta.length() || !MessageDigest.isEqual(sha.digest(), meta.sha256()) ||
+                    !SigV4.hex(md5.digest()).equals(meta.etag()))
+                    report.problem("Object checksum mismatch: " + path);
+            }
+        } catch (NoSuchFileException error) {
+            report.changedDuringScan++;
+        } catch (IOException | RuntimeException error) {
+            report.problem("Unreadable object record: " + path + " (" + error.getClass().getSimpleName() + ")");
+        }
+    }
+
+    private static void inspectUpload(Path dir, Report report) throws IOException {
+        if (!Files.isDirectory(dir)) {
+            report.problem("Unexpected multipart entry: " + dir);
+            return;
+        }
+        report.uploads++;
+        if (!Files.isRegularFile(dir.resolve("manifest"))) report.problem("Missing multipart manifest: " + dir);
+        try (var files = Files.list(dir)) {
+            for (Path path : files.toList()) {
+                String name = path.getFileName().toString();
+                if (name.matches("part-[0-9]{5}")) {
+                    try { report.stagedBytes += Files.size(path); }
+                    catch (NoSuchFileException error) { report.changedDuringScan++; }
+                } else if (!name.equals("manifest")) report.problem("Unexpected multipart file: " + path);
+            }
+        } catch (NoSuchFileException error) {
+            report.changedDuringScan++;
+        }
+    }
+
+    private static MessageDigest digest(String name) {
+        try { return MessageDigest.getInstance(name); }
+        catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); }
+    }
+}

src/cloud/lunarsky/store/ClusterJoin.java

@@ -0,0 +1,29 @@
+package cloud.lunarsky.store;
+
+import java.net.URI;
+import java.sql.DriverManager;
+import java.util.Map;
+import java.util.UUID;
+
+public final class ClusterJoin {
+    private ClusterJoin() {}
+
+    public static void main(String[] args) throws Exception {
+        if (args.length != 2)
+            throw new IllegalArgumentException("Usage: objectstore cluster-join node-url expected-host-uuid");
+        Map<String, String> env = System.getenv();
+        if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV")))
+            throw new IllegalArgumentException("Node registration is only enabled in local cluster mode");
+        URI url = URI.create(args[0]);
+        UUID host = UUID.fromString(args[1]);
+        try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"))) {
+            if (SchemaMigrator.prepare(connection, env.get("S3_BUCKET")) != 2)
+                throw new IllegalStateException("Migrate legacy replicas before joining nodes");
+            NodeClient.Node node = NodeRegistry.join(connection, url, host, env.get("CLUSTER_TOKEN"));
+            System.out.println("node_id=" + node.id());
+            System.out.println("host_id=" + node.hostId());
+            System.out.println("endpoint=" + node.url());
+        }
+    }
+}

src/cloud/lunarsky/store/ClusterMigrate.java

@@ -0,0 +1,157 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+public final class ClusterMigrate {
+    private ClusterMigrate() {}
+
+    public static void main(String[] args) throws Exception {
+        if ((args.length != 1 || !args[0].equals("--check")) &&
+            (args.length != 2 || !args[0].equals("--apply")))
+            throw new IllegalArgumentException("Usage: objectstore cluster-migrate --check | --apply node-id-0,node-id-1,node-id-2");
+        Map<String, String> env = System.getenv();
+        if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV")))
+            throw new IllegalArgumentException("Migration is enabled only in local cluster mode");
+        List<URI> urls = Arrays.stream(env.get("CLUSTER_NODES").split(",", -1)).map(URI::create).toList();
+        if (urls.size() != 3) throw new IllegalArgumentException("Legacy migration requires the original three URLs in their original order");
+        String token = env.get("CLUSTER_TOKEN");
+        List<NodeClient.Node> addresses = new ArrayList<>();
+        for (URI url : urls) {
+            NodeIdentity identity = NodeClient.probe(url, token);
+            addresses.add(new NodeClient.Node(identity.nodeId(), identity.hostId(), url));
+        }
+        NodeClient nodes = new NodeClient(addresses, token, null);
+        if (args[0].equals("--apply")) {
+            String actual = addresses.stream().map(node -> node.id().toString())
+                .collect(java.util.stream.Collectors.joining(","));
+            if (!actual.equals(args[1]))
+                throw new IllegalArgumentException("Confirmed legacy node mapping differs from the current ordered node identities");
+        }
+        try (Connection connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"))) {
+            int format = SchemaMigrator.prepare(connection, env.get("S3_BUCKET"));
+            System.out.println("cluster_format=" + format);
+            if (format == 2) return;
+            for (int index = 0; index < nodes.count(); index++) {
+                NodeClient.Node node = nodes.node(index);
+                System.out.println("legacy_" + index + "=" + node.id() + " host=" + node.hostId() +
+                    " endpoint=" + node.url());
+            }
+            long verified = verifyLiveSegments(connection, nodes);
+            System.out.println("live_segments_verified=" + verified);
+            if (args[0].equals("--check")) return;
+            apply(connection, nodes);
+            System.out.println("cluster_format=2");
+        }
+    }
+
+    private static long verifyLiveSegments(Connection connection, NodeClient nodes) throws SQLException, IOException {
+        long verified = 0;
+        try (PreparedStatement query = connection.prepareStatement(
+                "SELECT s.segment_id, s.length, s.sha256, s.replicas FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation");
+             ResultSet result = query.executeQuery()) {
+            while (result.next()) {
+                UUID segment = (UUID) result.getObject(1);
+                int length = result.getInt(2);
+                byte[] hash = result.getBytes(3);
+                for (int index : legacyIndices(result.getString(4), nodes.count())) {
+                    try {
+                        nodes.get(index, segment, length, hash);
+                    } catch (IOException offlineOrCorrupt) {
+                        throw new IOException("A listed live replica is unavailable or corrupt: segment " +
+                            segment + " legacy node " + index, offlineOrCorrupt);
+                    }
+                }
+                verified++;
+            }
+        }
+        return verified;
+    }
+
+    private static List<Integer> legacyIndices(String text, int count) throws IOException {
+        if (text == null || text.isBlank()) throw new IOException("Missing legacy replica list");
+        List<Integer> indices = new ArrayList<>();
+        Set<Integer> unique = new HashSet<>();
+        for (String part : text.split(",", -1)) {
+            int index;
+            try { index = Integer.parseInt(part); }
+            catch (NumberFormatException error) { throw new IOException("Invalid legacy replica index", error); }
+            if (index < 0 || index >= count || !unique.add(index))
+                throw new IOException("Invalid or duplicate legacy replica index");
+            indices.add(index);
+        }
+        return indices;
+    }
+
+    private static void apply(Connection connection, NodeClient nodes) throws SQLException, IOException {
+        try {
+            connection.setAutoCommit(false);
+            try (Statement statement = connection.createStatement()) {
+                statement.execute("SELECT pg_advisory_xact_lock(6834071092781)");
+                try (ResultSet result = statement.executeQuery("SELECT version FROM cluster_format WHERE singleton=1 FOR UPDATE")) {
+                    if (!result.next() || result.getInt(1) != 1)
+                        throw new IOException("Cluster format changed during migration");
+                }
+                try (ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_nodes)")) {
+                    result.next();
+                    if (result.getBoolean(1)) throw new IOException("Legacy migration already has node bindings");
+                }
+            }
+            try (PreparedStatement insert = connection.prepareStatement(
+                    "INSERT INTO cluster_nodes (node_id, host_id, endpoint, legacy_index, state) VALUES (?, ?, ?, ?, 'active')")) {
+                for (int index = 0; index < nodes.count(); index++) {
+                    NodeClient.Node node = nodes.node(index);
+                    insert.setObject(1, node.id());
+                    insert.setObject(2, node.hostId());
+                    insert.setString(3, node.url().toString());
+                    insert.setInt(4, index);
+                    insert.addBatch();
+                }
+                insert.executeBatch();
+            }
+            try (PreparedStatement select = connection.prepareStatement(
+                     "SELECT generation, ordinal, replicas FROM cluster_segments WHERE replica_ids IS NULL");
+                 ResultSet result = select.executeQuery();
+                 PreparedStatement update = connection.prepareStatement(
+                     "UPDATE cluster_segments SET replica_ids=? WHERE generation=? AND ordinal=? AND replica_ids IS NULL")) {
+                while (result.next()) {
+                    List<UUID> ids = new ArrayList<>();
+                    for (int index : legacyIndices(result.getString(3), nodes.count()))
+                        ids.add(nodes.node(index).id());
+                    update.setArray(1, connection.createArrayOf("uuid", ids.toArray()));
+                    update.setObject(2, result.getObject(1));
+                    update.setInt(3, result.getInt(2));
+                    if (update.executeUpdate() != 1) throw new IOException("Segment changed during migration");
+                }
+            }
+            try (Statement statement = connection.createStatement()) {
+                try (ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL)")) {
+                    result.next();
+                    if (result.getBoolean(1)) throw new IOException("Unconverted legacy segments remain");
+                }
+                statement.executeUpdate("UPDATE cluster_format SET version=2 WHERE singleton=1 AND version=1");
+                statement.execute("ALTER TABLE cluster_segments VALIDATE CONSTRAINT cluster_replica_ids_required");
+            }
+            connection.commit();
+        } catch (SQLException | IOException | RuntimeException error) {
+            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
+            if (error instanceof IOException io) throw io;
+            if (error instanceof SQLException sql) throw sql;
+            throw (RuntimeException) error;
+        } finally { connection.setAutoCommit(true); }
+    }
+}

src/cloud/lunarsky/store/ClusterNode.java

@@ -0,0 +1,244 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+
+/** Serves verified object segments to authenticated cluster peers. */
+public final class ClusterNode implements AutoCloseable {
+    static final int MAX_SEGMENT = 8 * 1024 * 1024;
+    private final Path root, segments, pending;
+    private final byte[] token;
+    private final byte[] repairToken;
+    private final NodeIdentity identity;
+    private final FileChannel lockChannel;
+    private final FileLock lock;
+
+    ClusterNode(Path root, String token, String repairToken, UUID hostId) throws IOException {
+        if (token == null || token.length() < 32) throw new IllegalArgumentException("Cluster token must have at least 32 characters");
+        if (repairToken == null || repairToken.length() < 32 || repairToken.equals(token))
+            throw new IllegalArgumentException("A separate repair token of at least 32 characters is required");
+        if (hostId == null) throw new IllegalArgumentException("Storage host ID is required");
+        this.root = root;
+        this.token = token.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        this.repairToken = repairToken.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        segments = root.resolve("segments");
+        pending = root.resolve("pending");
+        Files.createDirectories(root);
+        lockChannel = FileChannel.open(root.resolve(".process.lock"), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+        FileLock acquired;
+        try { acquired = lockChannel.tryLock(); }
+        catch (OverlappingFileLockException error) {
+            lockChannel.close();
+            throw new IOException("Node data directory is already in use", error);
+        }
+        if (acquired == null) {
+            lockChannel.close();
+            throw new IOException("Node data directory is already in use");
+        }
+        lock = acquired;
+        try {
+            identity = NodeIdentity.open(root, hostId);
+            Files.createDirectories(segments);
+            Files.createDirectories(pending);
+            DiskStore.syncDirectory(root);
+            try (var files = Files.list(pending)) {
+                for (Path file : files.toList()) {
+                    if (!Files.isRegularFile(file)) throw new IOException("Invalid pending entry: " + file);
+                    Files.delete(file);
+                }
+            }
+            DiskStore.syncDirectory(pending);
+        } catch (IOException error) {
+            close();
+            throw error;
+        }
+    }
+
+    void handle(HttpExchange exchange) throws IOException {
+        try {
+            String path = exchange.getRequestURI().getPath();
+            if (path.equals("/health") && exchange.getRequestMethod().equals("GET")) {
+                respond(exchange, 200, "ok");
+                return;
+            }
+            byte[] supplied = exchange.getRequestHeaders().getFirst("X-Cluster-Token") == null
+                ? new byte[0] : exchange.getRequestHeaders().getFirst("X-Cluster-Token")
+                    .getBytes(java.nio.charset.StandardCharsets.UTF_8);
+            if (!MessageDigest.isEqual(token, supplied)) {
+                respond(exchange, 403, "Forbidden");
+                return;
+            }
+            if (path.equals("/identity") && exchange.getRequestMethod().equals("GET")) {
+                respond(exchange, 200, identity.nodeId() + " " + identity.hostId());
+                return;
+            }
+            if (!identity.nodeId().toString().equals(exchange.getRequestHeaders().getFirst("X-Cluster-Expected-Node"))) {
+                respond(exchange, 409, "Wrong storage node");
+                return;
+            }
+            if (exchange.getRequestMethod().equals("PUT") &&
+                "true".equals(exchange.getRequestHeaders().getFirst("X-Cluster-Repair"))) {
+                String suppliedRepair = exchange.getRequestHeaders().getFirst("X-Cluster-Repair-Token");
+                byte[] suppliedBytes = suppliedRepair == null ? new byte[0]
+                    : suppliedRepair.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+                if (!MessageDigest.isEqual(repairToken, suppliedBytes)) {
+                    respond(exchange, 403, "Repair authority required");
+                    return;
+                }
+            }
+            if (!path.matches("/segments/[0-9a-f-]{36}")) {
+                respond(exchange, 404, "Not found");
+                return;
+            }
+            String id = path.substring("/segments/".length());
+            if (!UUID.fromString(id).toString().equals(id)) {
+                respond(exchange, 400, "Invalid segment ID");
+                return;
+            }
+            switch (exchange.getRequestMethod()) {
+                case "PUT" -> put(exchange, segmentPath(id, true));
+                case "GET" -> get(exchange, segmentPath(id, false));
+                default -> respond(exchange, 405, "Method not allowed");
+            }
+        } catch (IllegalArgumentException error) {
+            respond(exchange, 400, "Invalid request");
+        } catch (IOException error) {
+            if (exchange.getResponseCode() == -1) respond(exchange, 500, "Storage failure");
+            throw error;
+        } finally {
+            exchange.close();
+        }
+    }
+
+    private synchronized Path segmentPath(String id, boolean createShard) throws IOException {
+        Path shard = segments.resolve(id.substring(0, 2));
+        if (createShard && !Files.isDirectory(shard)) {
+            Files.createDirectories(shard);
+            DiskStore.syncDirectory(segments);
+        }
+        return shard.resolve(id);
+    }
+
+    private void put(HttpExchange exchange, Path target) throws IOException {
+        String hash = exchange.getRequestHeaders().getFirst("X-Cluster-Sha256");
+        String lengthText = exchange.getRequestHeaders().getFirst("Content-Length");
+        if (hash == null || !hash.matches("[0-9a-f]{64}") || lengthText == null) {
+            respond(exchange, 400, "Missing checksum or length");
+            return;
+        }
+        long length = Long.parseLong(lengthText);
+        if (length < 1 || length > MAX_SEGMENT) {
+            respond(exchange, 413, "Segment too large");
+            return;
+        }
+        Path temp = Files.createTempFile(pending, "segment-", ".part");
+        try {
+            MessageDigest digest = MessageDigest.getInstance("SHA-256");
+            long count = 0;
+            try (InputStream input = exchange.getRequestBody(); OutputStream output = Files.newOutputStream(temp)) {
+                byte[] buffer = new byte[65536];
+                int read;
+                while ((read = input.read(buffer)) != -1) {
+                    count += read;
+                    if (count > length) {
+                        respond(exchange, 400, "Body longer than declared length");
+                        return;
+                    }
+                    digest.update(buffer, 0, read);
+                    output.write(buffer, 0, read);
+                }
+            }
+            if (count != length || !hash.equals(HexFormat.of().formatHex(digest.digest()))) {
+                respond(exchange, 400, "Segment checksum mismatch");
+                return;
+            }
+            try (FileChannel channel = FileChannel.open(temp, StandardOpenOption.WRITE)) {
+                channel.force(true);
+            }
+            synchronized (this) {
+                if (Files.exists(target)) {
+                    byte[] existing = Files.readAllBytes(target);
+                    if (!Arrays.equals(existing, Files.readAllBytes(temp))) {
+                        if (!"true".equals(exchange.getRequestHeaders().getFirst("X-Cluster-Repair")) ||
+                            hash.equals(HexFormat.of().formatHex(SigV4.hash(existing)))) {
+                            respond(exchange, 409, "Segment ID conflict");
+                            return;
+                        }
+                        Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+                        DiskStore.syncDirectory(target.getParent());
+                    }
+                } else {
+                    Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
+                    DiskStore.syncDirectory(target.getParent());
+                }
+            }
+            respond(exchange, 200, "ok");
+        } catch (java.security.NoSuchAlgorithmException error) {
+            throw new IllegalStateException(error);
+        } finally {
+            Files.deleteIfExists(temp);
+        }
+    }
+
+    private void get(HttpExchange exchange, Path target) throws IOException {
+        if (!Files.isRegularFile(target)) {
+            respond(exchange, 404, "Segment not found");
+            return;
+        }
+        long length = Files.size(target);
+        if (length > MAX_SEGMENT) throw new IOException("Segment exceeds maximum length");
+        exchange.sendResponseHeaders(200, length);
+        try (InputStream input = Files.newInputStream(target)) {
+            input.transferTo(exchange.getResponseBody());
+        }
+    }
+
+    private static void respond(HttpExchange exchange, int status, String message) throws IOException {
+        byte[] body = message.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        exchange.getResponseHeaders().set("Content-Type", "text/plain; charset=utf-8");
+        exchange.sendResponseHeaders(status, body.length);
+        exchange.getResponseBody().write(body);
+    }
+
+    @Override public void close() throws IOException {
+        lock.release();
+        lockChannel.close();
+    }
+
+    public static void main(String[] args) throws Exception {
+        Map<String, String> env = System.getenv();
+        String token = env.get("CLUSTER_TOKEN");
+        var node = new ClusterNode(Path.of(env.getOrDefault("DATA_DIR", "/data")), token,
+            env.get("CLUSTER_REPAIR_TOKEN"),
+            UUID.fromString(env.get("CLUSTER_HOST_ID")));
+        int port = Integer.parseInt(env.getOrDefault("NODE_PORT", "9100"));
+        var server = HttpServer.create(new InetSocketAddress(env.getOrDefault("NODE_BIND", "127.0.0.1"), port), 64);
+        var executor = Executors.newVirtualThreadPerTaskExecutor();
+        server.setExecutor(executor);
+        server.createContext("/", node::handle);
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            server.stop(5);
+            executor.close();
+            try { node.close(); } catch (IOException error) { System.err.println("Node close failed: " + error); }
+        }));
+        server.start();
+        System.out.println("ObjectStore cluster node listening on :" + port);
+    }
+}

src/cloud/lunarsky/store/ClusterRepair.java

@@ -0,0 +1,26 @@
+package cloud.lunarsky.store;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Map;
+
+public final class ClusterRepair {
+    public static void main(String[] args) throws Exception {
+        if (args.length != 0) throw new IllegalArgumentException("Usage: objectstore cluster-repair");
+        Map<String, String> env = System.getenv();
+        if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV")))
+            throw new IllegalArgumentException("Cluster repair is only enabled in local cluster mode");
+        try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"),
+                env.get("POSTGRES_PASSWORD"), env.get("S3_BUCKET"),
+                Arrays.stream(env.get("CLUSTER_NODES").split(",")).map(URI::create).toList(),
+                env.get("CLUSTER_TOKEN"), env.get("CLUSTER_REPAIR_TOKEN"), 134217728, 2147483648L,
+                "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS")))) {
+            var report = store.repairOnce();
+            System.out.println("segments_scanned=" + report.scanned());
+            System.out.println("replicas_restored=" + report.restored());
+            System.out.println("segments_under_replicated=" + report.underReplicated());
+            System.out.println("segments_unrecoverable=" + report.unrecoverable());
+            if (report.unrecoverable() > 0) System.exit(1);
+        }
+    }
+}

src/cloud/lunarsky/store/ClusterStore.java

@@ -0,0 +1,442 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.DigestInputStream;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HexFormat;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+final class ClusterStore implements ObjectStorage {
+    private record Segment(UUID id, int length, byte[] hash, List<UUID> replicas) {}
+    private record RepairTarget(UUID generation, int ordinal, long version, Segment segment) {}
+    record RepairReport(int scanned, int restored, int underReplicated, int unrecoverable) {}
+    private final String jdbcUrl, user, password, configuredBucket;
+    private final NodeClient nodes;
+    private final long maxObject, maxTotal;
+    private final boolean testNodeDomains;
+
+    ClusterStore(String jdbcUrl, String user, String password, String bucket,
+                 List<URI> nodeUrls, String token, String repairToken, long maxObject, long maxTotal,
+                 boolean testNodeDomains) throws IOException {
+        if (jdbcUrl == null || !jdbcUrl.startsWith("jdbc:postgresql://") || user == null || password == null)
+            throw new IllegalArgumentException("Invalid metadata database configuration");
+        this.jdbcUrl = jdbcUrl; this.user = user; this.password = password;
+        this.configuredBucket = bucket; this.maxObject = maxObject; this.maxTotal = maxTotal;
+        this.testNodeDomains = testNodeDomains;
+        try (Connection connection = connect()) {
+            int format = SchemaMigrator.prepare(connection, bucket);
+            if (format != 2) throw new IOException("Legacy replica positions require objectstore cluster-migrate before this gateway can start");
+            nodes = NodeRegistry.load(connection, nodeUrls, token, repairToken);
+        } catch (SQLException error) { throw databaseError(error); }
+    }
+
+    private Connection connect() throws SQLException { return DriverManager.getConnection(jdbcUrl, user, password); }
+
+    @Override public Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
+                                  String checksum, boolean createOnly, String contentType) throws IOException {
+        if (!configuredBucket.equals(bucket)) throw new StoreException(404, "NoSuchBucket", "Bucket not found");
+        if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required");
+        if (length > maxObject) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit");
+        if (!nodes.availableHostsAtLeast(2, testNodeDomains))
+            throw new StoreException(503, "SlowDown", "Fewer than two storage hosts are available");
+        if (contentType.getBytes(java.nio.charset.StandardCharsets.UTF_8).length > 255)
+            throw new StoreException(400, "InvalidArgument", "Content-Type is too long");
+        MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
+        List<Segment> segments = new ArrayList<>();
+        byte[] fullHash;
+        Path staged = Files.createTempFile("objectstore-cluster-", ".pending");
+        try {
+            try (OutputStream output = Files.newOutputStream(staged)) {
+                byte[] buffer = new byte[65536];
+                long remaining = length;
+                while (remaining > 0) {
+                    int count = input.read(buffer, 0, (int) Math.min(buffer.length, remaining));
+                    if (count < 0) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length");
+                    if (count == 0) continue;
+                    sha.update(buffer, 0, count); md5.update(buffer, 0, count);
+                    output.write(buffer, 0, count);
+                    remaining -= count;
+                }
+            }
+            if (input.read() != -1) throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size");
+            fullHash = sha.digest();
+            if (!HexFormat.of().formatHex(fullHash).equals(expectedHash))
+                throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch");
+            if (checksum != null && !Base64.getEncoder().encodeToString(fullHash).equals(checksum))
+                throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch");
+            try (Connection connection = connect()) {
+                long previous = currentLength(connection, bucket, key);
+                if (createOnly && previous >= 0)
+                    throw new StoreException(412, "PreconditionFailed", "Object already exists");
+                try (PreparedStatement query = connection.prepareStatement("SELECT used_bytes FROM cluster_usage WHERE bucket=?")) {
+                    query.setString(1, bucket);
+                    try (ResultSet result = query.executeQuery()) {
+                        if (!result.next()) throw new SQLException("Bucket quota row is missing");
+                        if (result.getLong(1) - Math.max(0, previous) > maxTotal - length)
+                            throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached");
+                    }
+                }
+            } catch (SQLException error) { throw databaseError(error); }
+            try (InputStream stagedInput = Files.newInputStream(staged)) {
+                long remaining = length;
+                while (remaining > 0) {
+                    int wanted = (int) Math.min(ClusterNode.MAX_SEGMENT, remaining);
+                    byte[] bytes = stagedInput.readNBytes(wanted);
+                    if (bytes.length != wanted) throw new IOException("Staged object was truncated");
+                    byte[] segmentHash = SigV4.hash(bytes);
+                    UUID id = UUID.randomUUID();
+                    List<UUID> replicas = new ArrayList<>();
+                    Set<UUID> acceptedHosts = new HashSet<>();
+                    for (int index : PlacementPolicy.candidates(id, nodes, testNodeDomains)) {
+                        UUID host = nodes.faultDomain(index, testNodeDomains);
+                        if (acceptedHosts.contains(host)) continue;
+                        try {
+                            nodes.put(index, id, bytes, segmentHash);
+                            replicas.add(nodes.node(index).id());
+                            acceptedHosts.add(host);
+                            if (acceptedHosts.size() == 3) break;
+                        } catch (IOException error) {
+                            System.err.println("Cluster node " + nodes.node(index).id() +
+                                " did not accept segment " + id + ": " + error.getMessage());
+                        }
+                    }
+                    if (acceptedHosts.size() < 2)
+                        throw new StoreException(503, "SlowDown", "Fewer than two storage hosts accepted the segment");
+                    segments.add(new Segment(id, wanted, segmentHash, List.copyOf(replicas)));
+                    remaining -= wanted;
+                }
+            }
+        } finally { Files.deleteIfExists(staged); }
+        Metadata metadata = new Metadata(length, Instant.now().toEpochMilli(),
+            HexFormat.of().formatHex(md5.digest()), fullHash, bucket, key, contentType);
+        UUID generation = UUID.randomUUID();
+        try (Connection connection = connect()) {
+            connection.setAutoCommit(false);
+            try {
+                long used = lockUsage(connection, bucket);
+                long previous = currentLength(connection, bucket, key);
+                if (createOnly && previous >= 0)
+                    throw new StoreException(412, "PreconditionFailed", "Object already exists");
+                if (used - Math.max(0, previous) > maxTotal - length)
+                    throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached");
+                try (PreparedStatement insert = connection.prepareStatement(
+                    "INSERT INTO cluster_segments (generation, ordinal, segment_id, length, sha256, replicas, replica_ids) VALUES (?, ?, ?, ?, ?, 'v2', ?)")) {
+                    for (int i = 0; i < segments.size(); i++) {
+                        Segment segment = segments.get(i);
+                        insert.setObject(1, generation); insert.setInt(2, i); insert.setObject(3, segment.id());
+                        insert.setInt(4, segment.length()); insert.setBytes(5, segment.hash());
+                        insert.setArray(6, connection.createArrayOf("uuid", segment.replicas().toArray()));
+                        insert.addBatch();
+                    }
+                    insert.executeBatch();
+                }
+                try (PreparedStatement update = connection.prepareStatement(
+                    "INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, length=EXCLUDED.length, modified=EXCLUDED.modified, etag=EXCLUDED.etag, sha256=EXCLUDED.sha256, content_type=EXCLUDED.content_type")) {
+                    bindObject(update, metadata, generation); update.executeUpdate();
+                }
+                try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) {
+                    update.setLong(1, used - Math.max(0, previous) + length); update.setString(2, bucket); update.executeUpdate();
+                }
+                try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) {
+                    delete.setString(1, bucket); delete.setString(2, key); delete.executeUpdate();
+                }
+                connection.commit();
+                return metadata;
+            } catch (SQLException | RuntimeException error) {
+                connection.rollback();
+                if (error instanceof SQLException sql) throw databaseError(sql);
+                throw error;
+            }
+        } catch (SQLException error) { throw databaseError(error); }
+    }
+
+    @Override public OpenObject open(String bucket, String key) throws IOException {
+        try (Connection connection = connect()) {
+            connection.setAutoCommit(false);
+            connection.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
+            try {
+                Metadata metadata;
+                UUID generation;
+                try (PreparedStatement query = connection.prepareStatement(
+                    "SELECT generation, length, modified, etag, sha256, content_type FROM cluster_objects WHERE bucket=? AND object_key=?")) {
+                    query.setString(1, bucket); query.setString(2, key);
+                    try (ResultSet result = query.executeQuery()) {
+                        if (!result.next()) throw new StoreException(404, "NoSuchKey", "Object not found");
+                        generation = (UUID) result.getObject(1);
+                        metadata = new Metadata(result.getLong(2), result.getLong(3), result.getString(4),
+                            result.getBytes(5), bucket, key, result.getString(6));
+                    }
+                }
+                List<Segment> parts = new ArrayList<>();
+                try (PreparedStatement query = connection.prepareStatement(
+                    "SELECT segment_id, length, sha256, replica_ids FROM cluster_segments WHERE generation=? ORDER BY ordinal")) {
+                    query.setObject(1, generation);
+                    try (ResultSet result = query.executeQuery()) {
+                        long total = 0;
+                        while (result.next()) {
+                            Segment segment = new Segment((UUID) result.getObject(1), result.getInt(2),
+                                result.getBytes(3), replicaIds(result, 4));
+                            total = Math.addExact(total, segment.length());
+                            parts.add(segment);
+                        }
+                        if (total != metadata.length()) throw new IOException("Incomplete object manifest");
+                    }
+                }
+                connection.commit();
+                return new OpenObject(metadata, verifiedObject(parts, metadata));
+            } catch (SQLException | RuntimeException | IOException error) {
+                connection.rollback();
+                if (error instanceof SQLException sql) throw databaseError(sql);
+                if (error instanceof IOException io) throw io;
+                throw error;
+            }
+        } catch (SQLException error) { throw databaseError(error); }
+    }
+
+    @Override public void delete(String bucket, String key) throws IOException {
+        try (Connection connection = connect()) {
+            connection.setAutoCommit(false);
+            try {
+                long used = lockUsage(connection, bucket);
+                long previous = currentLength(connection, bucket, key);
+                try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_objects WHERE bucket=? AND object_key=?")) {
+                    delete.setString(1, bucket); delete.setString(2, key); delete.executeUpdate();
+                }
+                try (PreparedStatement update = connection.prepareStatement(
+                    "INSERT INTO cluster_tombstones VALUES (?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, deleted_at=EXCLUDED.deleted_at")) {
+                    update.setString(1, bucket); update.setString(2, key);
+                    update.setObject(3, UUID.randomUUID()); update.setLong(4, Instant.now().toEpochMilli());
+                    update.executeUpdate();
+                }
+                if (previous >= 0) {
+                    try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) {
+                        update.setLong(1, used - previous); update.setString(2, bucket); update.executeUpdate();
+                    }
+                }
+                connection.commit();
+            } catch (SQLException | RuntimeException error) {
+                connection.rollback();
+                if (error instanceof SQLException sql) throw databaseError(sql);
+                throw error;
+            }
+        } catch (SQLException error) { throw databaseError(error); }
+    }
+
+    @Override public ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) throws IOException {
+        List<ListedObject> entries = new ArrayList<>();
+        List<String> prefixes = new ArrayList<>();
+        if (maxKeys == 0) return new ListPage(entries, prefixes, null, false);
+        String lastKey = null, activePrefix = null;
+        boolean truncated = false;
+        try (Connection connection = connect()) {
+            connection.setAutoCommit(false);
+            try (PreparedStatement query = connection.prepareStatement(
+                "SELECT object_key, length, modified, etag, sha256, content_type FROM cluster_objects WHERE bucket=? AND object_key>=? ORDER BY object_key")) {
+                query.setString(1, bucket);
+                query.setString(2, after != null && after.compareTo(prefix) > 0 ? after : prefix);
+                query.setFetchSize(128);
+                try (ResultSet result = query.executeQuery()) {
+                    while (result.next()) {
+                        String key = result.getString(1);
+                        if (!key.startsWith(prefix)) break;
+                        if (after != null && key.compareTo(after) <= 0) continue;
+                        String group = null;
+                        if (!delimiter.isEmpty()) {
+                            int at = key.indexOf(delimiter, prefix.length());
+                            if (at >= 0) group = key.substring(0, at + delimiter.length());
+                        }
+                        if (group != null && group.equals(activePrefix)) { lastKey = key; continue; }
+                        if (entries.size() + prefixes.size() >= maxKeys) { truncated = true; break; }
+                        if (group != null) { prefixes.add(group); activePrefix = group; }
+                        else {
+                            entries.add(new ListedObject(key, new Metadata(result.getLong(2), result.getLong(3),
+                                result.getString(4), result.getBytes(5), bucket, key, result.getString(6))));
+                            activePrefix = null;
+                        }
+                        lastKey = key;
+                    }
+                }
+            }
+            connection.commit();
+        } catch (SQLException error) { throw databaseError(error); }
+        return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated);
+    }
+
+    private long lockUsage(Connection connection, String bucket) throws SQLException {
+        try (PreparedStatement query = connection.prepareStatement("SELECT used_bytes FROM cluster_usage WHERE bucket=? FOR UPDATE")) {
+            query.setString(1, bucket);
+            try (ResultSet result = query.executeQuery()) {
+                if (!result.next()) throw new SQLException("Bucket quota row is missing");
+                return result.getLong(1);
+            }
+        }
+    }
+    private long currentLength(Connection connection, String bucket, String key) throws SQLException {
+        try (PreparedStatement query = connection.prepareStatement("SELECT length FROM cluster_objects WHERE bucket=? AND object_key=?")) {
+            query.setString(1, bucket); query.setString(2, key);
+            try (ResultSet result = query.executeQuery()) { return result.next() ? result.getLong(1) : -1; }
+        }
+    }
+    private static void bindObject(PreparedStatement update, Metadata data, UUID generation) throws SQLException {
+        update.setString(1, data.bucket()); update.setString(2, data.key()); update.setObject(3, generation);
+        update.setLong(4, data.length()); update.setLong(5, data.modified()); update.setString(6, data.etag());
+        update.setBytes(7, data.sha256()); update.setString(8, data.contentType());
+    }
+    private static List<UUID> replicaIds(ResultSet result, int column) throws SQLException, IOException {
+        java.sql.Array value = result.getArray(column);
+        if (value == null) throw new IOException("Segment has no migrated replica identities");
+        try {
+            Object[] ids = (Object[]) value.getArray();
+            List<UUID> replicas = new ArrayList<>(ids.length);
+            for (Object id : ids) replicas.add((UUID) id);
+            return List.copyOf(replicas);
+        } finally { value.free(); }
+    }
+    private static IOException databaseError(SQLException error) { return new IOException("Metadata database operation failed", error); }
+    private static MessageDigest digest(String algorithm) {
+        try { return MessageDigest.getInstance(algorithm); }
+        catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); }
+    }
+    private InputStream verifiedObject(List<Segment> segments, Metadata metadata) throws IOException {
+        Path staged = Files.createTempFile("objectstore-read-", ".pending");
+        boolean ready = false;
+        try {
+            MessageDigest hash = digest("SHA-256");
+            long count;
+            try (InputStream source = new DigestInputStream(new SegmentStream(segments), hash);
+                 OutputStream output = Files.newOutputStream(staged)) {
+                count = source.transferTo(output);
+            }
+            if (count != metadata.length() || !MessageDigest.isEqual(hash.digest(), metadata.sha256()))
+                throw new IOException("Object manifest failed integrity verification");
+            InputStream file = Files.newInputStream(staged);
+            ready = true;
+            return new FilterInputStream(file) {
+                @Override public void close() throws IOException {
+                    try { super.close(); }
+                    finally { Files.deleteIfExists(staged); }
+                }
+            };
+        } finally { if (!ready) Files.deleteIfExists(staged); }
+    }
+    @Override public boolean ready() {
+        if (!nodes.availableHostsAtLeast(2, testNodeDomains)) return false;
+        try (Connection connection = connect(); var statement = connection.createStatement();
+             ResultSet result = statement.executeQuery("SELECT 1")) {
+            return result.next() && result.getInt(1) == 1;
+        } catch (SQLException error) { return false; }
+    }
+    RepairReport repairOnce() throws IOException {
+        int scanned = 0, restored = 0, underReplicated = 0, unrecoverable = 0;
+        try (Connection reader = connect()) {
+            reader.setAutoCommit(false);
+            try (PreparedStatement query = reader.prepareStatement(
+                "SELECT s.generation, s.ordinal, s.segment_id, s.length, s.sha256, s.replica_ids, s.placement_version FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation ORDER BY s.generation, s.ordinal")) {
+                query.setFetchSize(128);
+                try (ResultSet result = query.executeQuery()) {
+                    while (result.next()) {
+                        scanned++;
+                        RepairTarget target = new RepairTarget((UUID) result.getObject(1), result.getInt(2),
+                            result.getLong(7), new Segment((UUID) result.getObject(3), result.getInt(4),
+                                result.getBytes(5), replicaIds(result, 6)));
+                        Segment segment = target.segment();
+                        byte[] copy = null;
+                        Set<UUID> healthy = new HashSet<>();
+                        Set<UUID> healthyHosts = new HashSet<>();
+                        for (UUID id : segment.replicas()) {
+                            int node = nodes.index(id);
+                            if (node < 0) continue;
+                            try {
+                                byte[] candidate = nodes.get(node, segment.id(), segment.length(), segment.hash());
+                                if (copy == null) copy = candidate;
+                                healthy.add(id);
+                                healthyHosts.add(nodes.faultDomain(node, testNodeDomains));
+                            } catch (IOException error) { }
+                        }
+                        if (copy == null) { unrecoverable++; continue; }
+                        for (int node : PlacementPolicy.candidates(segment.id(), nodes, testNodeDomains)) {
+                            UUID host = nodes.faultDomain(node, testNodeDomains);
+                            if (healthyHosts.contains(host)) continue;
+                            try {
+                                nodes.repair(node, segment.id(), copy, segment.hash());
+                                healthy.add(nodes.node(node).id());
+                                healthyHosts.add(host);
+                                restored++;
+                            } catch (IOException error) { }
+                            if (healthyHosts.size() == 3) break;
+                        }
+                        if (healthyHosts.size() < 3) underReplicated++;
+                        Set<UUID> listed = new java.util.LinkedHashSet<>(segment.replicas());
+                        listed.addAll(healthy);
+                        if (listed.size() != segment.replicas().size()) {
+                            try (Connection writer = connect(); PreparedStatement update = writer.prepareStatement(
+                                "UPDATE cluster_segments SET replica_ids=?, placement_version=placement_version+1 WHERE generation=? AND ordinal=? AND placement_version=?")) {
+                                update.setArray(1, writer.createArrayOf("uuid", listed.toArray()));
+                                update.setObject(2, target.generation());
+                                update.setInt(3, target.ordinal());
+                                update.setLong(4, target.version());
+                                update.executeUpdate();
+                            }
+                        }
+                    }
+                }
+            }
+            reader.commit();
+        } catch (SQLException error) { throw databaseError(error); }
+        return new RepairReport(scanned, restored, underReplicated, unrecoverable);
+    }
+    @Override public void close() {}
+
+    private final class SegmentStream extends InputStream {
+        private final List<Segment> segments;
+        private int position;
+        private ByteArrayInputStream current;
+        private boolean closed;
+        SegmentStream(List<Segment> segments) { this.segments = segments; }
+        @Override public int read() throws IOException {
+            byte[] one = new byte[1];
+            int count = read(one, 0, 1);
+            return count < 0 ? -1 : one[0] & 255;
+        }
+        @Override public int read(byte[] buffer, int offset, int length) throws IOException {
+            if (closed) throw new IOException("Object stream is closed");
+            if (length == 0) return 0;
+            while (current == null || current.available() == 0) {
+                if (position == segments.size()) return -1;
+                Segment segment = segments.get(position++);
+                IOException failure = null;
+                for (UUID replica : segment.replicas()) {
+                    int node = nodes.index(replica);
+                    if (node < 0) continue;
+                    try {
+                        byte[] bytes = nodes.get(node, segment.id(), segment.length(), segment.hash());
+                        current = new ByteArrayInputStream(bytes);
+                        break;
+                    } catch (IOException error) { failure = error; }
+                }
+                if (current == null || current.available() == 0)
+                    throw new StoreException(503, "SlowDown", "No verified replica is currently available", failure);
+            }
+            return current.read(buffer, offset, length);
+        }
+        @Override public void close() { closed = true; current = null; }
+    }
+}

src/cloud/lunarsky/store/DiskStore.java

@@ -5,28 +5,35 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.charset.CodingErrorAction;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.*;
 import java.security.MessageDigest;
 import java.time.Instant;
-import java.util.Arrays;
-import java.util.HexFormat;
+import java.util.*;
+import cloud.lunarsky.store.ObjectStorage.Metadata;
+import cloud.lunarsky.store.ObjectStorage.OpenObject;
+import cloud.lunarsky.store.ObjectStorage.ListedObject;
+import cloud.lunarsky.store.ObjectStorage.ListPage;
 
-final class DiskStore implements AutoCloseable {
-    private static final long MAGIC = 0x4c534f424a303031L;
-    private static final int HEADER = 72;
-    private final Path objects, temporary;
+final class DiskStore implements ObjectStorage {
+    private static final long MAGIC_V1 = 0x4c534f424a303031L;
+    private static final long MAGIC_V2 = 0x4c534f424a303032L;
+    private static final int HEADER_V1 = 72;
+    private static final int HEADER_V2 = 78;
+    private final Path root, objects, temporary;
     private final FileChannel lockChannel;
     private final FileLock processLock;
     private final long maxObject, maxTotal;
     private final Object[] locks = new Object[128];
+    private final NavigableMap<String, Metadata> index = new TreeMap<>();
     private long used;
-    record Metadata(long length, long modified, String etag, byte[] sha256) {}
-    record OpenObject(Metadata metadata, InputStream stream) implements AutoCloseable {
-        public void close() throws IOException { stream.close(); }
-    }
+    private long objectCount, legacyCount;
+
+    record Record(Metadata metadata, int headerLength) {}
 
     DiskStore(Path root, long maxObject, long maxTotal) throws IOException {
+        this.root = root;
         objects = root.resolve("objects"); temporary = root.resolve("pending");
         this.maxObject = maxObject; this.maxTotal = maxTotal;
         Arrays.setAll(locks, i -> new Object());
@@ -39,16 +46,25 @@ final class DiskStore implements AutoCloseable {
             catch (OverlappingFileLockException e) { throw new IOException("Data directory is already in use", e); }
             if (acquired == null) throw new IOException("Data directory is already in use");
             Files.createDirectories(objects); Files.createDirectories(temporary);
+            syncDirectory(root);
             try (var paths = Files.list(temporary)) {
                 for (Path p : paths.toList()) if (p.getFileName().toString().endsWith(".part")) Files.delete(p);
             }
             try (var paths = Files.walk(objects)) {
                 for (Path p : paths.filter(Files::isRegularFile).toList()) {
-                    try(var in=new DataInputStream(Files.newInputStream(p))) {
-                        long length = metadata(in).length();
-                        if(Files.size(p)-HEADER != length) throw new IOException("Truncated or oversized object record: "+p);
-                        used = Math.addExact(used, length);
-                    }
+                    Record record;
+                    try (var in = new DataInputStream(Files.newInputStream(p))) { record = readRecord(in); }
+                    Metadata meta = record.metadata();
+                    if (Files.size(p) - record.headerLength() != meta.length())
+                        throw new IOException("Truncated or oversized object record: " + p);
+                    if (meta.key() != null) {
+                        if (!p.equals(objectPath(meta.bucket(), meta.key())))
+                            throw new IOException("Mismatched object record: " + p);
+                        if (index.put(indexKey(meta.bucket(), meta.key()), meta) != null)
+                            throw new IOException("Duplicate object record: " + p);
+                    } else legacyCount++;
+                    objectCount++;
+                    used = Math.addExact(used, meta.length());
                 }
             }
             ready = true;
@@ -66,72 +82,189 @@ final class DiskStore implements AutoCloseable {
         processLock.release();
         lockChannel.close();
     }
+    Path root() { return root; }
+    long maxObject() { return maxObject; }
+    long maxTotal() { return maxTotal; }
+    synchronized long usedBytes() { return used; }
+    synchronized int indexedObjects() { return index.size(); }
+    synchronized long objectCount() { return objectCount; }
+    synchronized long legacyObjects() { return legacyCount; }
 
-    private Path object(String bucket, String key) throws IOException {
+    private static String indexKey(String bucket, String key) { return bucket + "\0" + key; }
+    private Path objectPath(String bucket, String key) {
         String id = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(StandardCharsets.UTF_8)));
-        Path shard=objects.resolve(id.substring(0,2));Files.createDirectories(shard);
-        return shard.resolve(id);
+        return objects.resolve(id.substring(0, 2)).resolve(id);
+    }
+    private synchronized Path object(String bucket, String key) throws IOException {
+        Path path = objectPath(bucket, key);
+        if (!Files.isDirectory(path.getParent())) {
+            Files.createDirectories(path.getParent());
+            syncDirectory(objects);
+        }
+        return path;
+    }
+    static void syncDirectory(Path directory) throws IOException {
+        try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) {
+            channel.force(true);
+        }
     }
     private Object lock(Path p) { return locks[(p.hashCode() & 0x7fffffff) % locks.length]; }
 
-    Metadata put(String bucket,String key,InputStream input,long length,String expectedHash,String checksum,boolean createOnly) throws IOException {
+    public Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
+                 String checksum, boolean createOnly, String contentType) throws IOException {
         if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required");
         if (length > maxObject) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit");
+        byte[] bucketBytes = bucket.getBytes(StandardCharsets.UTF_8);
+        byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+        byte[] typeBytes = contentType.getBytes(StandardCharsets.UTF_8);
+        if (bucketBytes.length > 63 || keyBytes.length > 1024 || typeBytes.length > 255)
+            throw new StoreException(400, "InvalidArgument", "Object metadata is too long");
+        int headerLength = HEADER_V2 + bucketBytes.length + keyBytes.length + typeBytes.length;
         Path destination = object(bucket, key), pending = Files.createTempFile(temporary, "upload-", ".part");
         try {
             MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
             long count = 0;
             try (OutputStream out = Files.newOutputStream(pending)) {
-                out.write(new byte[HEADER]);byte[] buffer=new byte[65536];int n;
-                while((n=input.read(buffer))!=-1){count+=n;if(count>length||count>maxObject)throw new StoreException(413,"EntityTooLarge","Payload exceeds declared size");sha.update(buffer,0,n);md5.update(buffer,0,n);out.write(buffer,0,n);}
+                out.write(new byte[headerLength]);
+                byte[] buffer = new byte[65536]; int n;
+                while ((n = input.read(buffer)) != -1) {
+                    count += n;
+                    if (count > length || count > maxObject)
+                        throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size");
+                    sha.update(buffer, 0, n); md5.update(buffer, 0, n); out.write(buffer, 0, n);
+                }
             }
             if (count != length) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length");
             byte[] hash = sha.digest(), etag = md5.digest();
-            if(!MessageDigest.isEqual(hash,HexFormat.of().parseHex(expectedHash)))throw new StoreException(400,"XAmzContentSHA256Mismatch","Payload hash mismatch");
-            if(checksum!=null&&!java.util.Base64.getEncoder().encodeToString(hash).equals(checksum))throw new StoreException(400,"BadDigest","SHA-256 checksum mismatch");
+            if (!MessageDigest.isEqual(hash, HexFormat.of().parseHex(expectedHash)))
+                throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch");
+            if (checksum != null && !Base64.getEncoder().encodeToString(hash).equals(checksum))
+                throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch");
             long modified = Instant.now().toEpochMilli();
+            ByteBuffer header = ByteBuffer.allocate(headerLength).putLong(MAGIC_V2).putLong(count)
+                .putLong(modified).put(etag).put(hash).putShort((short) bucketBytes.length)
+                .putShort((short) keyBytes.length).putShort((short) typeBytes.length)
+                .put(bucketBytes).put(keyBytes).put(typeBytes);
+            header.flip();
             try (FileChannel file = FileChannel.open(pending, StandardOpenOption.WRITE)) {
-                ByteBuffer header=ByteBuffer.allocate(HEADER).putLong(MAGIC).putLong(count).putLong(modified).put(etag).put(hash);header.flip();
-                while(header.hasRemaining())file.write(header);file.force(true);
+                while (header.hasRemaining()) file.write(header, header.position());
+                file.force(true);
             }
+            Metadata metadata = new Metadata(count, modified, SigV4.hex(etag), hash, bucket, key, contentType);
             synchronized (lock(destination)) {
                 long previous = 0;
-                if(Files.exists(destination)){
+                boolean existed = Files.exists(destination);
+                boolean legacy = false;
+                if (existed) {
                     if (createOnly) throw new StoreException(412, "PreconditionFailed", "Object already exists");
-                    try(var in=new DataInputStream(Files.newInputStream(destination))){previous=metadata(in).length();}
+                    try (var in = new DataInputStream(Files.newInputStream(destination))) {
+                        Metadata old = readRecord(in).metadata();
+                        previous = old.length();
+                        legacy = old.key() == null;
+                    }
                 }
                 synchronized (this) {
-                    if(used-previous+count>maxTotal)throw new StoreException(507,"InsufficientStorage","Store capacity limit reached");
+                    if (used - previous + count > maxTotal)
+                        throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached");
                     Files.move(pending, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
                     used = used - previous + count;
+                    if (!existed) objectCount++;
+                    if (legacy) legacyCount--;
+                    index.put(indexKey(bucket, key), metadata);
+                    syncDirectory(destination.getParent());
                 }
             }
-            return new Metadata(count,modified,SigV4.hex(etag),hash);
+            return metadata;
         } finally { Files.deleteIfExists(pending); }
     }
 
-    OpenObject open(String bucket,String key) throws IOException {
+    public OpenObject open(String bucket, String key) throws IOException {
         Path destination = object(bucket, key);
         synchronized (lock(destination)) {
             final DataInputStream input;
             try { input = new DataInputStream(Files.newInputStream(destination)); }
             catch (NoSuchFileException e) { throw new StoreException(404, "NoSuchKey", "Object not found"); }
-            try{return new OpenObject(metadata(input),input);}catch(IOException e){input.close();throw e;}
+            try { return new OpenObject(readRecord(input).metadata(), input); }
+            catch (IOException e) { input.close(); throw e; }
         }
     }
-    void delete(String bucket,String key) throws IOException {
+
+    public void delete(String bucket, String key) throws IOException {
         Path destination = object(bucket, key);
         synchronized (lock(destination)) {
             if (!Files.exists(destination)) return;
-            long length;try(var input=new DataInputStream(Files.newInputStream(destination))){length=metadata(input).length();}
-            synchronized(this){Files.delete(destination);used-=length;}
+            long length;
+            boolean legacy;
+            try (var input = new DataInputStream(Files.newInputStream(destination))) {
+                Metadata old = readRecord(input).metadata();
+                length = old.length();
+                legacy = old.key() == null;
+            }
+            synchronized (this) {
+                Files.delete(destination);
+                used -= length;
+                objectCount--;
+                if (legacy) legacyCount--;
+                index.remove(indexKey(bucket, key));
+                syncDirectory(destination.getParent());
             }
         }
-    private static Metadata metadata(DataInputStream in) throws IOException {
-        if(in.readLong()!=MAGIC)throw new IOException("Invalid object record");
-        long length=in.readLong(),modified=in.readLong();byte[] md5=new byte[16],sha=new byte[32];in.readFully(md5);in.readFully(sha);
-        if(length<0)throw new IOException("Invalid object length");
-        return new Metadata(length,modified,SigV4.hex(md5),sha);
     }
-    private static MessageDigest digest(String algorithm){try{return MessageDigest.getInstance(algorithm);}catch(java.security.NoSuchAlgorithmException e){throw new IllegalStateException(e);}}
+
+    public synchronized ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) {
+        List<ListedObject> entries = new ArrayList<>();
+        List<String> prefixes = new ArrayList<>();
+        if (maxKeys == 0) return new ListPage(entries, prefixes, null, false);
+        String lastKey = null;
+        boolean truncated = false;
+        String activePrefix = null;
+        for (Metadata meta : index.values()) {
+            if (!meta.bucket().equals(bucket) || !meta.key().startsWith(prefix)) continue;
+            String key = meta.key();
+            if (after != null && key.compareTo(after) <= 0) continue;
+            String group = null;
+            if (!delimiter.isEmpty()) {
+                int at = key.indexOf(delimiter, prefix.length());
+                if (at >= 0) group = key.substring(0, at + delimiter.length());
+            }
+            if (group != null && group.equals(activePrefix)) { lastKey = key; continue; }
+            if (entries.size() + prefixes.size() >= maxKeys) { truncated = true; break; }
+            if (group != null) { prefixes.add(group); activePrefix = group; }
+            else { entries.add(new ListedObject(key, meta)); activePrefix = null; }
+            lastKey = key;
+        }
+        return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated);
+    }
+
+    static Record readRecord(DataInputStream in) throws IOException {
+        long magic = in.readLong();
+        if (magic != MAGIC_V1 && magic != MAGIC_V2) throw new IOException("Invalid object record");
+        long length = in.readLong(), modified = in.readLong();
+        byte[] md5 = new byte[16], sha = new byte[32];
+        in.readFully(md5); in.readFully(sha);
+        if (length < 0) throw new IOException("Invalid object record length");
+        if (magic == MAGIC_V1)
+            return new Record(new Metadata(length, modified, SigV4.hex(md5), sha,
+                null, null, "application/octet-stream"), HEADER_V1);
+        int bucketLength = in.readUnsignedShort(), keyLength = in.readUnsignedShort(), typeLength = in.readUnsignedShort();
+        if (bucketLength < 1 || bucketLength > 63 || keyLength < 1 || keyLength > 1024 || typeLength < 1 || typeLength > 255)
+            throw new IOException("Invalid object record metadata");
+        String bucket = utf8(in.readNBytes(bucketLength));
+        String key = utf8(in.readNBytes(keyLength));
+        String contentType = utf8(in.readNBytes(typeLength));
+        if (bucket.getBytes(StandardCharsets.UTF_8).length != bucketLength ||
+            key.getBytes(StandardCharsets.UTF_8).length != keyLength ||
+            contentType.getBytes(StandardCharsets.UTF_8).length != typeLength)
+            throw new IOException("Invalid object record metadata");
+        return new Record(new Metadata(length, modified, SigV4.hex(md5), sha,
+            bucket, key, contentType), HEADER_V2 + bucketLength + keyLength + typeLength);
+    }
+    private static String utf8(byte[] bytes) throws IOException {
+        return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
+            .decode(ByteBuffer.wrap(bytes)).toString();
+    }
+    private static MessageDigest digest(String algorithm) {
+        try { return MessageDigest.getInstance(algorithm); }
+        catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
+    }
 }

src/cloud/lunarsky/store/Main.java

@@ -3,24 +3,40 @@ package cloud.lunarsky.store;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpServer;
 import java.io.IOException;
+import java.io.ByteArrayInputStream;
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
+import java.util.Base64;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Semaphore;
 
 public final class Main {
-    private final DiskStore store;
+    private final ObjectStorage store;
     private final SigV4 authentication;
     private final String bucket;
+    private final MultipartStorage multipart;
     private final Semaphore slots = new Semaphore(16);
-    Main(DiskStore store,SigV4 authentication,String bucket){this.store=store;this.authentication=authentication;this.bucket=bucket;}
+
+    Main(DiskStore store, SigV4 authentication, String bucket) throws IOException {
+        this(store, new MultipartStore(store), authentication, bucket);
+    }
+
+    Main(ObjectStorage store, MultipartStorage multipart, SigV4 authentication, String bucket) {
+        this.store = store; this.multipart = multipart;
+        this.authentication = authentication; this.bucket = bucket;
+    }
 
     void handle(HttpExchange exchange) throws IOException {
         boolean admitted = slots.tryAcquire();
@@ -31,84 +47,454 @@ public final class Main {
             if (!admitted) throw new StoreException(503, "SlowDown", "Too many concurrent requests");
             if (exchange.getRequestURI().getRawPath().equals("/health") && exchange.getRequestMethod().equals("GET")) {
                 byte[] body = "{\"status\":\"ok\",\"service\":\"lunarsky-objectstore\"}".getBytes(StandardCharsets.UTF_8);
-                exchange.getResponseHeaders().set("Content-Type","application/json");exchange.sendResponseHeaders(200,body.length);exchange.getResponseBody().write(body);return;
+                exchange.getResponseHeaders().set("Content-Type", "application/json");
+                exchange.sendResponseHeaders(200, body.length);
+                exchange.getResponseBody().write(body);
+                return;
+            }
+            if (exchange.getRequestURI().getRawPath().equals("/ready") && exchange.getRequestMethod().equals("GET")) {
+                boolean ready = store.ready();
+                byte[] body = (ready ? "ready" : "unavailable").getBytes(StandardCharsets.UTF_8);
+                exchange.getResponseHeaders().set("Content-Type", "text/plain; charset=utf-8");
+                exchange.sendResponseHeaders(ready ? 200 : 503, body.length);
+                exchange.getResponseBody().write(body);
+                return;
             }
             String hash = authentication.verify(exchange.getRequestMethod(), exchange.getRequestURI(), exchange.getRequestHeaders());
             String path = SigV4.decode(exchange.getRequestURI().getRawPath());
+            Map<String, String> query = query(exchange.getRequestURI().getRawQuery());
+            if (path.equals("/" + bucket) || path.equals("/" + bucket + "/")) {
+                if (!exchange.getRequestMethod().equals("GET") || !"2".equals(query.get("list-type")) ||
+                    !query.keySet().stream().allMatch(java.util.Set.of("list-type", "prefix", "delimiter", "max-keys",
+                        "continuation-token", "start-after", "encoding-type", "x-id")::contains) ||
+                    (query.containsKey("x-id") && !"ListObjectsV2".equals(query.get("x-id"))))
+                    unsupported("Bucket operation");
+                requireEmptyBody(exchange, hash);
+                listObjects(exchange, query);
+                return;
+            }
             String prefix = "/" + bucket + "/";
             if (!path.startsWith(prefix)) throw new StoreException(404, "NoSuchBucket", "Bucket not found");
             String key = path.substring(prefix.length());
-            if(key.isEmpty()||key.getBytes(StandardCharsets.UTF_8).length>1024||key.indexOf('\0')>=0)throw new StoreException(400,"InvalidArgument","Invalid object key");
-            String query=exchange.getRequestURI().getRawQuery();
-            if(query!=null&&!query.isEmpty()&&!query.matches("x-id=(PutObject|GetObject|HeadObject|DeleteObject)"))unsupported("Query operation");
+            if (key.isEmpty() || key.getBytes(StandardCharsets.UTF_8).length > 1024 || key.indexOf('\0') >= 0)
+                throw new StoreException(400, "InvalidArgument", "Invalid object key");
+            String method = exchange.getRequestMethod();
+            boolean multipartRequest = multipartRequest(method, query);
+            if (!multipartRequest && !query.isEmpty() && !(query.size() == 1 &&
+                ("PutObject".equals(query.get("x-id")) || "GetObject".equals(query.get("x-id")) ||
+                 "HeadObject".equals(query.get("x-id")) || "DeleteObject".equals(query.get("x-id")))))
+                unsupported("Query operation");
             var headers = exchange.getRequestHeaders();
-            if(headers.containsKey("range")||headers.containsKey("if-match")||headers.containsKey("if-modified-since")||headers.containsKey("if-unmodified-since"))unsupported("Range or conditional read");
             for (String name : headers.keySet()) {
                 String lower = name.toLowerCase(java.util.Locale.ROOT);
-                if(lower.startsWith("x-amz-")&&!java.util.Set.of("x-amz-date","x-amz-content-sha256","x-amz-checksum-sha256","x-amz-sdk-checksum-algorithm","x-amz-user-agent").contains(lower))unsupported("Amazon header");
-                if(lower.startsWith("x-amz-meta-")||lower.startsWith("x-amz-server-side-")||lower.startsWith("x-amz-copy-")||lower.startsWith("x-amz-acl")||lower.startsWith("x-amz-grant")||lower.startsWith("x-amz-tagging")||lower.equals("content-md5"))unsupported("Object metadata, encryption, ACL, copy, tagging or MD5 header");
-                if(lower.startsWith("x-amz-checksum-")&&!lower.equals("x-amz-checksum-sha256"))unsupported("Checksum algorithm");
+                if (lower.startsWith("x-amz-") && !java.util.Set.of("x-amz-date", "x-amz-content-sha256",
+                    "x-amz-checksum-sha256", "x-amz-sdk-checksum-algorithm", "x-amz-user-agent").contains(lower))
+                    unsupported("Amazon header");
+                if (lower.startsWith("x-amz-meta-") || lower.startsWith("x-amz-server-side-") ||
+                    lower.startsWith("x-amz-copy-") || lower.startsWith("x-amz-acl") ||
+                    lower.startsWith("x-amz-grant") || lower.startsWith("x-amz-tagging") ||
+                    lower.equals("content-md5")) unsupported("Object metadata, encryption, ACL, copy, tagging or MD5 header");
+                if (lower.startsWith("x-amz-checksum-") && !lower.equals("x-amz-checksum-sha256"))
+                    unsupported("Checksum algorithm");
             }
-            String method=exchange.getRequestMethod();
             String algorithm = SigV4.single(headers, "x-amz-sdk-checksum-algorithm");
             if (algorithm != null && !algorithm.equals("SHA256")) unsupported("Checksum algorithm");
-            if(!method.equals("PUT")&&(headers.containsKey("transfer-encoding")||(headers.containsKey("content-length")&&!"0".equals(SigV4.single(headers,"content-length")))))throw new StoreException(400,"InvalidRequest","Read/delete requests must have empty bodies");
-            if(!method.equals("PUT")&&!hash.equals(SigV4.hex(SigV4.hash(new byte[0]))))throw new StoreException(400,"InvalidRequest","Read/delete requests must have empty bodies");
+            if (multipartRequest) {
+                handleMultipart(exchange, method, query, key, hash);
+                return;
+            }
+            if (!method.equals("PUT")) requireEmptyBody(exchange, hash);
             switch (method) {
                 case "PUT" -> {
                     String length = SigV4.single(headers, "content-length"), condition = SigV4.single(headers, "if-none-match");
                     if (condition != null && !condition.equals("*")) unsupported("Write condition");
-                    long bytes;try{bytes=length==null?-1:Long.parseLong(length);}catch(NumberFormatException e){throw new StoreException(400,"InvalidArgument","Invalid Content-Length");}
+                    long bytes;
+                    try { bytes = length == null ? -1 : Long.parseLong(length); }
+                    catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid Content-Length"); }
                     if (headers.containsKey("content-encoding")) unsupported("Encoded payload");
-                    DiskStore.Metadata data=store.put(bucket,key,exchange.getRequestBody(),bytes,hash,SigV4.single(headers,"x-amz-checksum-sha256"),condition!=null);
+                    String contentType = contentType(headers);
+                    ObjectStorage.Metadata data = store.put(bucket, key, exchange.getRequestBody(), bytes, hash,
+                        SigV4.single(headers, "x-amz-checksum-sha256"), condition != null, contentType);
                     exchange.getResponseHeaders().set("ETag", "\"" + data.etag() + "\"");
-                    exchange.getResponseHeaders().set("x-amz-checksum-sha256",java.util.Base64.getEncoder().encodeToString(data.sha256()));
+                    exchange.getResponseHeaders().set("x-amz-checksum-sha256", Base64.getEncoder().encodeToString(data.sha256()));
                     exchange.sendResponseHeaders(200, -1);
                 }
-                case "GET", "HEAD" -> {
-                    if(headers.containsKey("if-none-match"))unsupported("Conditional read");
-                    try(var object=store.open(bucket,key)){
-                        var meta=object.metadata();
-                        exchange.getResponseHeaders().set("Content-Type","application/octet-stream");
-                        exchange.getResponseHeaders().set("Content-Length",Long.toString(meta.length()));
-                        exchange.getResponseHeaders().set("ETag","\""+meta.etag()+"\"");
-                        exchange.getResponseHeaders().set("Last-Modified",DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC).format(Instant.ofEpochMilli(meta.modified())));
-                        if(method.equals("HEAD")||meta.length()==0)exchange.sendResponseHeaders(200,-1);
-                        else{exchange.sendResponseHeaders(200,meta.length());object.stream().transferTo(exchange.getResponseBody());}
+                case "GET", "HEAD" -> readObject(exchange, key);
+                case "DELETE" -> {
+                    if (headers.containsKey("if-none-match")) unsupported("Conditional delete");
+                    store.delete(bucket, key);
+                    exchange.sendResponseHeaders(204, -1);
                 }
-                }
-                case "DELETE" -> {if(headers.containsKey("if-none-match"))unsupported("Conditional delete");store.delete(bucket,key);exchange.sendResponseHeaders(204,-1);}
                 default -> unsupported("HTTP method");
             }
         } catch (StoreException error) { sendError(exchange, error.status, error.code, error.getMessage(), requestId); }
-        catch(Exception error){System.err.println("ObjectStore request failed: "+requestId+" "+error.getClass().getSimpleName());sendError(exchange,500,"InternalError","Storage operation failed",requestId);}
-        finally {if(admitted)slots.release();exchange.close();}
+        catch (Exception error) {
+            System.err.println("ObjectStore request failed: " + requestId + " " + error.getClass().getSimpleName());
+            sendError(exchange, 500, "InternalError", "Storage operation failed", requestId);
+        } finally { if (admitted) slots.release(); exchange.close(); }
+    }
+
+    private static String contentType(com.sun.net.httpserver.Headers headers) {
+        String value = SigV4.single(headers, "content-type");
+        if (value == null) return "application/octet-stream";
+        if (value.isBlank() || value.getBytes(StandardCharsets.UTF_8).length > 255 ||
+            !value.chars().allMatch(c -> c >= 32 && c <= 126))
+            throw new StoreException(400, "InvalidArgument", "Invalid Content-Type");
+        return value;
+    }
+
+    private static void requireEmptyBody(HttpExchange exchange, String hash) {
+        var headers = exchange.getRequestHeaders();
+        if (headers.containsKey("transfer-encoding") ||
+            (headers.containsKey("content-length") && !"0".equals(SigV4.single(headers, "content-length"))) ||
+            !hash.equals(SigV4.hex(SigV4.hash(new byte[0]))))
+            throw new StoreException(400, "InvalidRequest", "Request must have an empty body");
+    }
+
+    private static boolean multipartRequest(String method, Map<String, String> query) {
+        if (query.containsKey("uploads"))
+            return method.equals("POST") && query.get("uploads").isEmpty() &&
+                query.keySet().stream().allMatch(java.util.Set.of("uploads", "x-id")::contains) &&
+                (!query.containsKey("x-id") || query.get("x-id").equals("CreateMultipartUpload"));
+        if (!query.containsKey("uploadId") ||
+            !query.keySet().stream().allMatch(java.util.Set.of("uploadId", "partNumber", "x-id")::contains))
+            return false;
+        String xId = query.get("x-id");
+        if (method.equals("PUT")) return query.containsKey("partNumber") &&
+            (xId == null || xId.equals("UploadPart"));
+        if (query.containsKey("partNumber")) return false;
+        return (method.equals("POST") && (xId == null || xId.equals("CompleteMultipartUpload"))) ||
+            (method.equals("DELETE") && (xId == null || xId.equals("AbortMultipartUpload")));
+    }
+
+    private void handleMultipart(HttpExchange exchange, String method, Map<String, String> query,
+                                 String key, String hash) throws IOException {
+        var headers = exchange.getRequestHeaders();
+        if (headers.containsKey("content-encoding") || headers.containsKey("if-none-match"))
+            unsupported("Multipart request header");
+        if (query.containsKey("uploads")) {
+            requireEmptyBody(exchange, hash);
+            String id = multipart.create(bucket, key, contentType(headers));
+            sendXml(exchange, 200, "<InitiateMultipartUploadResult><Bucket>" + xml(bucket) +
+                "</Bucket><Key>" + xml(key) + "</Key><UploadId>" + id +
+                "</UploadId></InitiateMultipartUploadResult>");
+            return;
+        }
+        String id = query.get("uploadId");
+        switch (method) {
+            case "PUT" -> {
+                int number;
+                try { number = Integer.parseInt(query.get("partNumber")); }
+                catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid part number"); }
+                long length = contentLength(headers);
+                String etag = multipart.putPart(id, bucket, key, number, exchange.getRequestBody(), length,
+                    hash, SigV4.single(headers, "x-amz-checksum-sha256"));
+                exchange.getResponseHeaders().set("ETag", "\"" + etag + "\"");
+                exchange.sendResponseHeaders(200, -1);
+            }
+            case "POST" -> {
+                byte[] body = signedBody(exchange, hash, 65536);
+                List<MultipartStorage.Part> parts = completedParts(body);
+                var meta = multipart.complete(id, bucket, key, parts);
+                sendXml(exchange, 200, "<CompleteMultipartUploadResult><Bucket>" + xml(bucket) +
+                    "</Bucket><Key>" + xml(key) + "</Key><ETag>&quot;" + meta.etag() +
+                    "&quot;</ETag></CompleteMultipartUploadResult>");
+            }
+            case "DELETE" -> {
+                requireEmptyBody(exchange, hash);
+                multipart.abort(id, bucket, key);
+                exchange.sendResponseHeaders(204, -1);
+            }
+            default -> unsupported("Multipart operation");
+        }
+    }
+
+    private static long contentLength(com.sun.net.httpserver.Headers headers) {
+        String text = SigV4.single(headers, "content-length");
+        if (text == null) return -1;
+        try { return Long.parseLong(text); }
+        catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid Content-Length"); }
+    }
+    private static byte[] signedBody(HttpExchange exchange, String hash, int limit) throws IOException {
+        long length = contentLength(exchange.getRequestHeaders());
+        if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required");
+        if (length > limit) throw new StoreException(413, "EntityTooLarge", "Request body is too large");
+        byte[] body = exchange.getRequestBody().readNBytes(limit + 1);
+        if (body.length != length) throw new StoreException(400, "IncompleteBody", "Body length does not match Content-Length");
+        if (!SigV4.hex(SigV4.hash(body)).equals(hash))
+            throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch");
+        return body;
+    }
+    private static List<MultipartStorage.Part> completedParts(byte[] body) {
+        try {
+            var factory = javax.xml.parsers.DocumentBuilderFactory.newInstance();
+            factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+            factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+            factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+            factory.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true);
+            factory.setExpandEntityReferences(false);
+            var document = factory.newDocumentBuilder().parse(new ByteArrayInputStream(body));
+            if (!document.getDocumentElement().getNodeName().equals("CompleteMultipartUpload"))
+                throw new IllegalArgumentException();
+            var nodes = document.getDocumentElement().getChildNodes();
+            List<MultipartStorage.Part> parts = new ArrayList<>();
+            for (int i = 0; i < nodes.getLength(); i++) {
+                if (!(nodes.item(i) instanceof org.w3c.dom.Element element)) continue;
+                if (!element.getTagName().equals("Part")) throw new IllegalArgumentException();
+                String number = null, etag = null;
+                var fields = element.getChildNodes();
+                for (int j = 0; j < fields.getLength(); j++) {
+                    if (!(fields.item(j) instanceof org.w3c.dom.Element field)) continue;
+                    if (field.getTagName().equals("PartNumber")) number = field.getTextContent().trim();
+                    else if (field.getTagName().equals("ETag")) etag = field.getTextContent().trim();
+                    else throw new IllegalArgumentException();
+                }
+                if (number == null || etag == null || !etag.matches("\"?[0-9a-f]{32}\"?"))
+                    throw new IllegalArgumentException();
+                parts.add(new MultipartStorage.Part(Integer.parseInt(number), etag));
+                if (parts.size() > 10000) throw new IllegalArgumentException();
+            }
+            return parts;
+        } catch (Exception error) {
+            throw new StoreException(400, "MalformedXML", "Invalid multipart completion body");
+        }
+    }
+    private static void sendXml(HttpExchange exchange, int status, String xml) throws IOException {
+        byte[] body = ("<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + xml).getBytes(StandardCharsets.UTF_8);
+        exchange.getResponseHeaders().set("Content-Type", "application/xml");
+        exchange.sendResponseHeaders(status, body.length);
+        exchange.getResponseBody().write(body);
+    }
+
+    private void readObject(HttpExchange exchange, String key) throws IOException {
+        var headers = exchange.getRequestHeaders();
+        if (headers.containsKey("if-match") || headers.containsKey("if-modified-since") ||
+            headers.containsKey("if-unmodified-since")) unsupported("Conditional read");
+        try (var object = store.open(bucket, key)) {
+            var meta = object.metadata();
+            String etag = "\"" + meta.etag() + "\"";
+            String noneMatch = SigV4.single(headers, "if-none-match");
+            if (noneMatch != null && (noneMatch.equals("*") ||
+                java.util.Arrays.stream(noneMatch.split(",")).map(String::trim).anyMatch(etag::equals))) {
+                exchange.getResponseHeaders().set("ETag", etag);
+                exchange.sendResponseHeaders(304, -1);
+                return;
+            }
+            Range range;
+            try { range = range(SigV4.single(headers, "range"), meta.length()); }
+            catch (StoreException error) {
+                if (error.status == 416) exchange.getResponseHeaders().set("Content-Range", "bytes */" + meta.length());
+                throw error;
+            }
+            var response = exchange.getResponseHeaders();
+            response.set("Content-Type", meta.contentType());
+            response.set("Content-Length", Long.toString(range.length()));
+            response.set("Accept-Ranges", "bytes");
+            response.set("ETag", etag);
+            response.set("Last-Modified", DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC)
+                .format(Instant.ofEpochMilli(meta.modified())));
+            if (range.partial()) response.set("Content-Range", "bytes " + range.start() + "-" + range.end() + "/" + meta.length());
+            int status = range.partial() ? 206 : 200;
+            if (exchange.getRequestMethod().equals("HEAD") || range.length() == 0)
+                exchange.sendResponseHeaders(status, -1);
+            else {
+                object.stream().skipNBytes(range.start());
+                exchange.sendResponseHeaders(status, range.length());
+                byte[] buffer = new byte[65536]; long left = range.length();
+                while (left > 0) {
+                    int n = object.stream().read(buffer, 0, (int) Math.min(buffer.length, left));
+                    if (n < 0) throw new IOException("Object body ended before its recorded length");
+                    exchange.getResponseBody().write(buffer, 0, n);
+                    left -= n;
+                }
+            }
+        }
+    }
+
+    private record Range(long start, long end, boolean partial) {
+        long length() { return end < start ? 0 : end - start + 1; }
+    }
+    private static Range range(String header, long size) {
+        if (header == null) return new Range(0, size - 1, false);
+        if (!header.matches("bytes=[0-9]*-[0-9]*") || header.equals("bytes=-") || size == 0)
+            throw new StoreException(416, "InvalidRange", "The requested range is not satisfiable");
+        String[] parts = header.substring(6).split("-", -1);
+        try {
+            long start, end;
+            if (parts[0].isEmpty()) {
+                long suffix = Long.parseLong(parts[1]);
+                if (suffix == 0) throw new NumberFormatException();
+                start = Math.max(0, size - suffix); end = size - 1;
+            } else {
+                start = Long.parseLong(parts[0]);
+                end = parts[1].isEmpty() ? size - 1 : Math.min(Long.parseLong(parts[1]), size - 1);
+            }
+            if (start >= size || end < start) throw new NumberFormatException();
+            return new Range(start, end, true);
+        } catch (NumberFormatException e) {
+            throw new StoreException(416, "InvalidRange", "The requested range is not satisfiable");
+        }
+    }
+
+    private void listObjects(HttpExchange exchange, Map<String, String> query) throws IOException {
+        String prefix = query.getOrDefault("prefix", ""), delimiter = query.getOrDefault("delimiter", "");
+        String encoding = query.get("encoding-type");
+        if (encoding != null && !encoding.equals("url")) unsupported("Encoding type");
+        if (query.containsKey("continuation-token") && query.containsKey("start-after"))
+            throw new StoreException(400, "InvalidArgument", "Use either continuation-token or start-after");
+        int maxKeys;
+        try { maxKeys = Integer.parseInt(query.getOrDefault("max-keys", "1000")); }
+        catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid max-keys"); }
+        if (maxKeys < 0 || maxKeys > 1000) throw new StoreException(400, "InvalidArgument", "Invalid max-keys");
+        String after = query.get("start-after");
+        if (query.containsKey("continuation-token")) {
+            try {
+                byte[] decoded = Base64.getUrlDecoder().decode(query.get("continuation-token"));
+                after = StandardCharsets.UTF_8.newDecoder().onMalformedInput(java.nio.charset.CodingErrorAction.REPORT)
+                    .decode(java.nio.ByteBuffer.wrap(decoded)).toString();
+            } catch (IllegalArgumentException | java.nio.charset.CharacterCodingException e) {
+                throw new StoreException(400, "InvalidArgument", "Invalid continuation token");
+            }
+        }
+        var page = store.list(bucket, prefix, delimiter, maxKeys, after);
+        StringBuilder xml = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">");
+        xml.append("<Name>").append(xml(bucket)).append("</Name><Prefix>").append(xml(listKey(prefix, encoding))).append("</Prefix>");
+        if (!delimiter.isEmpty()) xml.append("<Delimiter>").append(xml(listKey(delimiter, encoding))).append("</Delimiter>");
+        if (encoding != null) xml.append("<EncodingType>url</EncodingType>");
+        if (query.containsKey("continuation-token")) xml.append("<ContinuationToken>")
+            .append(xml(query.get("continuation-token"))).append("</ContinuationToken>");
+        if (query.containsKey("start-after")) xml.append("<StartAfter>")
+            .append(xml(listKey(query.get("start-after"), encoding))).append("</StartAfter>");
+        xml.append("<KeyCount>").append(page.keyCount()).append("</KeyCount><MaxKeys>").append(maxKeys)
+            .append("</MaxKeys><IsTruncated>").append(page.truncated()).append("</IsTruncated>");
+        int objectAt = 0, prefixAt = 0;
+        while (objectAt < page.objects().size() || prefixAt < page.prefixes().size()) {
+            if (objectAt < page.objects().size() &&
+                (prefixAt == page.prefixes().size() ||
+                 page.objects().get(objectAt).key().compareTo(page.prefixes().get(prefixAt)) < 0)) {
+                var entry = page.objects().get(objectAt++);
+                var meta = entry.metadata();
+                xml.append("<Contents><Key>").append(xml(listKey(entry.key(), encoding))).append("</Key><LastModified>")
+                    .append(Instant.ofEpochMilli(meta.modified())).append("</LastModified><ETag>&quot;")
+                    .append(meta.etag()).append("&quot;</ETag><Size>").append(meta.length())
+                    .append("</Size><StorageClass>STANDARD</StorageClass></Contents>");
+            } else {
+                xml.append("<CommonPrefixes><Prefix>")
+                    .append(xml(listKey(page.prefixes().get(prefixAt++), encoding)))
+                    .append("</Prefix></CommonPrefixes>");
+            }
+        }
+        if (page.truncated()) xml.append("<NextContinuationToken>")
+            .append(Base64.getUrlEncoder().withoutPadding().encodeToString(page.nextKey().getBytes(StandardCharsets.UTF_8)))
+            .append("</NextContinuationToken>");
+        xml.append("</ListBucketResult>");
+        byte[] body = xml.toString().getBytes(StandardCharsets.UTF_8);
+        exchange.getResponseHeaders().set("Content-Type", "application/xml");
+        exchange.sendResponseHeaders(200, body.length);
+        exchange.getResponseBody().write(body);
+    }
+
+    private static String listKey(String key, String encoding) {
+        return encoding == null ? key : SigV4.encode(key, false);
+    }
+    private static Map<String, String> query(String raw) {
+        Map<String, String> result = new HashMap<>();
+        if (raw == null || raw.isEmpty()) return result;
+        for (String part : raw.split("&", -1)) {
+            String[] pair = part.split("=", 2);
+            String name = SigV4.decode(pair[0]);
+            String value = SigV4.decode(pair.length == 2 ? pair[1] : "");
+            if (result.put(name, value) != null)
+                throw new StoreException(400, "InvalidArgument", "Duplicate query parameter");
+        }
+        return result;
+    }
+    private static void unsupported(String feature) { throw new StoreException(501, "NotImplemented", feature + " is not supported"); }
+    private static String xml(String text) {
+        return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;");
     }
-    private static void unsupported(String feature){throw new StoreException(501,"NotImplemented",feature+" is not supported in this prototype");}
-    private static String xml(String text){return text.replace("&","&amp;").replace("<","&lt;").replace(">","&gt;").replace("\"","&quot;");}
     private static void sendError(HttpExchange exchange, int status, String code, String message, String id) throws IOException {
         if (exchange.getResponseCode() != -1) return;
-        byte[] body=("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>"+xml(code)+"</Code><Message>"+xml(message)+"</Message><RequestId>"+id+"</RequestId></Error>").getBytes(StandardCharsets.UTF_8);
-        exchange.getResponseHeaders().set("Content-Type","application/xml");exchange.sendResponseHeaders(status,exchange.getRequestMethod().equals("HEAD")?-1:body.length);
+        byte[] body = ("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>" + xml(code) +
+            "</Code><Message>" + xml(message) + "</Message><RequestId>" + id + "</RequestId></Error>")
+            .getBytes(StandardCharsets.UTF_8);
+        exchange.getResponseHeaders().set("Content-Type", "application/xml");
+        exchange.sendResponseHeaders(status, exchange.getRequestMethod().equals("HEAD") ? -1 : body.length);
         if (!exchange.getRequestMethod().equals("HEAD")) exchange.getResponseBody().write(body);
     }
     public static void main(String[] args) throws Exception {
         Map<String, String> env = System.getenv();
-        String access=required(env,"S3_ACCESS_KEY"),secret=required(env,"S3_SECRET_KEY"),bucket=env.getOrDefault("S3_BUCKET","lunaris-files"),region=env.getOrDefault("S3_REGION","us-east-1");
-        if(!access.matches("[A-Za-z0-9]{16,128}")||secret.length()<32||!bucket.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]"))throw new IllegalArgumentException("Invalid storage credentials/bucket configuration");
-        long maxObject=Long.parseLong(env.getOrDefault("MAX_OBJECT_BYTES","10485760")),maxTotal=Long.parseLong(env.getOrDefault("MAX_TOTAL_BYTES","2147483648"));
-        if(maxObject<1||maxObject>1073741824L||maxTotal<maxObject)throw new IllegalArgumentException("Invalid size limits");
-        var store=new DiskStore(Path.of(env.getOrDefault("DATA_DIR","/data")),maxObject,maxTotal);
-        var app=new Main(store,new SigV4(access,secret,region,Clock.systemUTC()),bucket);
-        var server=HttpServer.create(new InetSocketAddress(Integer.parseInt(env.getOrDefault("PORT","9000"))),64);
-        var executor=Executors.newVirtualThreadPerTaskExecutor();server.setExecutor(executor);server.createContext("/",app::handle);
+        String access = required(env, "S3_ACCESS_KEY"), secret = required(env, "S3_SECRET_KEY");
+        String bucket = env.getOrDefault("S3_BUCKET", "lunaris-files"), region = env.getOrDefault("S3_REGION", "us-east-1");
+        if (!access.matches("[A-Za-z0-9]{16,128}") || secret.length() < 32 ||
+            !bucket.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]"))
+            throw new IllegalArgumentException("Invalid storage credentials/bucket configuration");
+        long maxObject = Long.parseLong(env.getOrDefault("MAX_OBJECT_BYTES", "134217728"));
+        long maxTotal = Long.parseLong(env.getOrDefault("MAX_TOTAL_BYTES", "2147483648"));
+        if (maxObject < 1 || maxObject > 1073741824L || maxTotal < maxObject)
+            throw new IllegalArgumentException("Invalid size limits");
+        String mode = env.getOrDefault("STORE_MODE", "disk");
+        ObjectStorage store;
+        MultipartStorage multipart;
+        if (mode.equals("cluster")) {
+            if (!"true".equals(env.get("CLUSTER_LOCAL_DEV")))
+                throw new IllegalArgumentException("Cluster mode is local development only; set CLUSTER_LOCAL_DEV=true");
+            String[] urls = required(env, "CLUSTER_NODES").split(",", -1);
+            if (urls.length < 2) throw new IllegalArgumentException("CLUSTER_NODES requires at least two URLs");
+            store = new ClusterStore(required(env, "POSTGRES_JDBC_URL"), required(env, "POSTGRES_USER"),
+                required(env, "POSTGRES_PASSWORD"), bucket,
+                java.util.Arrays.stream(urls).map(URI::create).toList(), required(env, "CLUSTER_TOKEN"), null,
+                maxObject, maxTotal, "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS")));
+            multipart = new UnavailableMultipart();
+        } else if (mode.equals("disk")) {
+            DiskStore disk = new DiskStore(Path.of(env.getOrDefault("DATA_DIR", "/data")), maxObject, maxTotal);
+            store = disk;
+            multipart = new MultipartStore(disk);
+        } else throw new IllegalArgumentException("Invalid STORE_MODE");
+        var app = new Main(store, multipart, new SigV4(access, secret, region, Clock.systemUTC()), bucket);
+        int port = Integer.parseInt(env.getOrDefault("PORT", "9000"));
+        var server = HttpServer.create(mode.equals("cluster")
+            ? new InetSocketAddress(env.getOrDefault("BIND_ADDRESS", "127.0.0.1"), port)
+            : new InetSocketAddress(port), 64);
+        var executor = Executors.newVirtualThreadPerTaskExecutor();
+        server.setExecutor(executor); server.createContext("/", app::handle);
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             server.stop(5);
             executor.close();
             try { store.close(); }
             catch (IOException error) { System.err.println("Could not release ObjectStore data lock: " + error.getMessage()); }
         }));
-        server.start();System.out.println("LunarSky ObjectStore listening; S3 object-operation prototype, bucket="+bucket);
+        server.start();
+        if (store instanceof DiskStore disk) printStartup(disk, app.multipart, bucket, region, port);
+        else System.out.println("ObjectStore local cluster prototype v" + Version.VALUE + " listening on :" + port);
+    }
+    private static void printStartup(DiskStore store, MultipartStorage multipart,
+                                     String bucket, String region, int port) {
+        System.out.println("       *");
+        System.out.println("      / \\       LUNARSKY");
+        System.out.println("     / L \\      ObjectStore");
+        System.out.println("    /_____\\     v" + Version.VALUE);
+        System.out.println();
+        System.out.println("  Bucket     " + bucket + " (" + region + ")");
+        System.out.println("  Objects    " + store.objectCount() + " total, " +
+            store.indexedObjects() + " indexed, " + size(store.usedBytes()) + " / " + size(store.maxTotal()));
+        if (store.legacyObjects() > 0)
+            System.out.println("  Legacy     " + store.legacyObjects() + " objects without stored keys");
+        System.out.println("  Multipart  " + multipart.activeUploads() + " active, " +
+            size(multipart.stagedBytes()) + " staged");
+        System.out.println("  Listening  :" + port);
+    }
+    private static String size(long bytes) {
+        if (bytes < 1024) return bytes + " B";
+        String[] units = {"KiB", "MiB", "GiB", "TiB"};
+        double value = bytes;
+        int unit = -1;
+        do { value /= 1024; unit++; } while (value >= 1024 && unit < units.length - 1);
+        return String.format(Locale.ROOT, "%.1f %s", value, units[unit]);
+    }
+    private static String required(Map<String, String> env, String key) {
+        String value = env.get(key);
+        if (value == null || value.isBlank()) throw new IllegalArgumentException("Missing " + key);
+        return value;
     }
-    private static String required(Map<String,String> env,String key){String value=env.get(key);if(value==null||value.isBlank())throw new IllegalArgumentException("Missing "+key);return value;}
 }

src/cloud/lunarsky/store/MultipartStorage.java

@@ -0,0 +1,17 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+interface MultipartStorage {
+    record Part(int number, String etag) {}
+
+    String create(String bucket, String key, String contentType) throws IOException;
+    String putPart(String id, String bucket, String key, int number, InputStream input,
+                   long length, String expectedHash, String checksum) throws IOException;
+    ObjectStorage.Metadata complete(String id, String bucket, String key, List<Part> parts) throws IOException;
+    void abort(String id, String bucket, String key) throws IOException;
+    int activeUploads();
+    long stagedBytes();
+}

src/cloud/lunarsky/store/MultipartStore.java

@@ -0,0 +1,217 @@
+package cloud.lunarsky.store;
+
+import java.io.*;
+import java.nio.file.*;
+import java.security.MessageDigest;
+import java.util.*;
+import cloud.lunarsky.store.MultipartStorage.Part;
+
+final class MultipartStore implements MultipartStorage {
+    private static final int MAGIC = 0x4c534d50;
+    private final DiskStore store;
+    private final Path root;
+    private long staged;
+    private int active;
+
+    private record Upload(String bucket, String key, String contentType) {}
+
+    MultipartStore(DiskStore store) throws IOException {
+        this.store = store;
+        root = store.root().resolve("multipart");
+        Files.createDirectories(root);
+        DiskStore.syncDirectory(store.root());
+        try (var uploads = Files.list(root)) {
+            for (Path dir : uploads.toList()) {
+                if (dir.getFileName().toString().startsWith(".creating-")) {
+                    discardCreating(dir);
+                    continue;
+                }
+                if (!Files.isDirectory(dir)) throw new IOException("Invalid multipart upload entry: " + dir);
+                readUpload(dir);
+                active++;
+                try (var files = Files.list(dir)) {
+                    for (Path file : files.toList()) {
+                        if (file.getFileName().toString().matches("part-[0-9]{5}"))
+                            staged = Math.addExact(staged, Files.size(file));
+                        else if (!file.getFileName().toString().equals("manifest"))
+                            throw new IOException("Invalid multipart upload entry: " + file);
+                    }
+                }
+            }
+        }
+        if (staged > store.maxTotal()) throw new IOException("Multipart staging limit exceeded");
+    }
+
+    public synchronized String create(String bucket, String key, String contentType) throws IOException {
+        if (active >= 32) throw new StoreException(503, "SlowDown", "Too many active uploads");
+        String id = UUID.randomUUID().toString();
+        Path pending = root.resolve(".creating-" + id), dir = root.resolve(id);
+        Files.createDirectory(pending);
+        try {
+            try (var output = new DataOutputStream(Files.newOutputStream(pending.resolve("manifest"), StandardOpenOption.CREATE_NEW))) {
+                output.writeInt(MAGIC);
+                output.writeUTF(bucket);
+                output.writeUTF(key);
+                output.writeUTF(contentType);
+            }
+            try (var channel = java.nio.channels.FileChannel.open(pending.resolve("manifest"), StandardOpenOption.READ)) {
+                channel.force(true);
+            }
+            DiskStore.syncDirectory(pending);
+            Files.move(pending, dir, StandardCopyOption.ATOMIC_MOVE);
+            DiskStore.syncDirectory(root);
+        } catch (IOException error) {
+            discardCreating(pending);
+            discardCreating(dir);
+            throw error;
+        }
+        active++;
+        return id;
+    }
+
+    private void discardCreating(Path dir) throws IOException {
+        if (!Files.exists(dir)) return;
+        if (Files.isDirectory(dir)) {
+            try (var files = Files.list(dir)) {
+                for (Path file : files.toList()) Files.delete(file);
+            }
+        }
+        Files.delete(dir);
+        DiskStore.syncDirectory(root);
+    }
+
+    public synchronized int activeUploads() { return active; }
+    public synchronized long stagedBytes() { return staged; }
+
+    public synchronized String putPart(String id, String bucket, String key, int number, InputStream input,
+                                long length, String expectedHash, String checksum) throws IOException {
+        if (number < 1 || number > 10000) throw new StoreException(400, "InvalidArgument", "Invalid part number");
+        if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required");
+        if (length > store.maxObject()) throw new StoreException(413, "EntityTooLarge", "Part exceeds the object limit");
+        Path dir = upload(id, bucket, key), target = part(dir, number);
+        long previous = Files.exists(target) ? Files.size(target) : 0;
+        if (staged - previous + length > store.maxTotal())
+            throw new StoreException(507, "InsufficientStorage", "Multipart staging limit reached");
+        Path pending = Files.createTempFile(store.root().resolve("pending"), "part-", ".part");
+        try {
+            MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
+            long count = 0;
+            try (var output = Files.newOutputStream(pending)) {
+                byte[] buffer = new byte[65536]; int n;
+                while ((n = input.read(buffer)) != -1) {
+                    count += n;
+                    if (count > length) throw new StoreException(413, "EntityTooLarge", "Part exceeds declared size");
+                    sha.update(buffer, 0, n); md5.update(buffer, 0, n); output.write(buffer, 0, n);
+                }
+            }
+            if (count != length) throw new StoreException(400, "IncompleteBody", "Part length does not match Content-Length");
+            byte[] actual = sha.digest();
+            if (!MessageDigest.isEqual(actual, HexFormat.of().parseHex(expectedHash)))
+                throw new StoreException(400, "XAmzContentSHA256Mismatch", "Part hash mismatch");
+            if (checksum != null && !Base64.getEncoder().encodeToString(actual).equals(checksum))
+                throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch");
+            try (var channel = java.nio.channels.FileChannel.open(pending, StandardOpenOption.WRITE)) { channel.force(true); }
+            Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+            staged = staged - previous + length;
+            DiskStore.syncDirectory(dir);
+            return SigV4.hex(md5.digest());
+        } finally { Files.deleteIfExists(pending); }
+    }
+
+    public synchronized ObjectStorage.Metadata complete(String id, String bucket, String key, List<Part> parts) throws IOException {
+        Path dir = upload(id, bucket, key);
+        if (parts.isEmpty() || parts.size() > 10000)
+            throw new StoreException(400, "InvalidPart", "No valid parts supplied");
+        MessageDigest sha = digest("SHA-256");
+        List<Path> paths = new ArrayList<>();
+        long total = 0; int last = 0;
+        for (Part part : parts) {
+            if (part.number() <= last || part.number() > 10000)
+                throw new StoreException(400, "InvalidPartOrder", "Parts must be in ascending order");
+            last = part.number();
+            Path file = part(dir, part.number());
+            if (!Files.isRegularFile(file)) throw new StoreException(400, "InvalidPart", "Missing part");
+            long length = Files.size(file);
+            total += length;
+            if (total > store.maxObject()) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit");
+            MessageDigest md5 = digest("MD5");
+            try (var input = Files.newInputStream(file)) {
+                byte[] buffer = new byte[65536]; int n;
+                while ((n = input.read(buffer)) != -1) { sha.update(buffer, 0, n); md5.update(buffer, 0, n); }
+            }
+            if (!SigV4.hex(md5.digest()).equals(part.etag().replace("\"", "")))
+                throw new StoreException(400, "InvalidPart", "Part ETag mismatch");
+            paths.add(file);
+        }
+        ObjectStorage.Metadata result;
+        try (InputStream input = new PartsInput(paths)) {
+            result = store.put(bucket, key, input, total, SigV4.hex(sha.digest()), null, false,
+                readUpload(dir).contentType());
+        }
+        remove(dir);
+        return result;
+    }
+
+    public synchronized void abort(String id, String bucket, String key) throws IOException {
+        remove(upload(id, bucket, key));
+    }
+
+    private Path upload(String id, String bucket, String key) throws IOException {
+        if (!id.matches("[0-9a-f-]{36}")) throw new StoreException(404, "NoSuchUpload", "Upload not found");
+        Path dir = root.resolve(id);
+        if (!Files.isDirectory(dir)) throw new StoreException(404, "NoSuchUpload", "Upload not found");
+        Upload upload = readUpload(dir);
+        if (!upload.bucket().equals(bucket) || !upload.key().equals(key))
+            throw new StoreException(404, "NoSuchUpload", "Upload not found");
+        return dir;
+    }
+    private static Upload readUpload(Path dir) throws IOException {
+        try (var input = new DataInputStream(Files.newInputStream(dir.resolve("manifest")))) {
+            if (input.readInt() != MAGIC) throw new IOException("Invalid multipart upload manifest");
+            Upload upload = new Upload(input.readUTF(), input.readUTF(), input.readUTF());
+            if (input.read() != -1) throw new IOException("Invalid multipart upload manifest");
+            return upload;
+        }
+    }
+    private static Path part(Path dir, int number) { return dir.resolve("part-%05d".formatted(number)); }
+    private void remove(Path dir) throws IOException {
+        long removed = 0;
+        try (var files = Files.list(dir)) {
+            for (Path file : files.toList()) {
+                if (file.getFileName().toString().matches("part-[0-9]{5}")) removed += Files.size(file);
+                Files.delete(file);
+            }
+        }
+        DiskStore.syncDirectory(dir);
+        Files.delete(dir);
+        DiskStore.syncDirectory(root);
+        staged -= removed;
+        active--;
+    }
+    private static MessageDigest digest(String algorithm) {
+        try { return MessageDigest.getInstance(algorithm); }
+        catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
+    }
+    private static final class PartsInput extends InputStream {
+        private final Iterator<Path> parts;
+        private InputStream current;
+        PartsInput(List<Path> files) { parts = files.iterator(); }
+        @Override public int read() throws IOException {
+            byte[] one = new byte[1];
+            return read(one, 0, 1) < 0 ? -1 : one[0] & 255;
+        }
+        @Override public int read(byte[] buffer, int offset, int length) throws IOException {
+            if (length == 0) return 0;
+            while (true) {
+                if (current == null) {
+                    if (!parts.hasNext()) return -1;
+                    current = Files.newInputStream(parts.next());
+                }
+                int n = current.read(buffer, offset, length);
+                if (n >= 0) return n;
+                current.close(); current = null;
+            }
+        }
+        @Override public void close() throws IOException { if (current != null) current.close(); }
+    }
+}

src/cloud/lunarsky/store/NodeClient.java

@@ -0,0 +1,147 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.util.HashSet;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+final class NodeClient {
+    record Node(UUID id, UUID hostId, URI url) {}
+
+    private static final HttpClient IDENTITY_HTTP = HttpClient.newBuilder()
+        .connectTimeout(Duration.ofSeconds(2)).build();
+    private final List<Node> nodes;
+    private final String token;
+    private final String repairToken;
+    private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(3)).build();
+
+    NodeClient(List<Node> nodes, String token, String repairToken) {
+        if (nodes.isEmpty() || nodes.stream().map(Node::id).distinct().count() != nodes.size() ||
+            nodes.stream().map(Node::url).distinct().count() != nodes.size())
+            throw new IllegalArgumentException("Cluster node IDs and URLs must be unique");
+        if (token == null || token.length() < 32) throw new IllegalArgumentException("Invalid cluster token");
+        for (Node node : nodes) {
+            if (node.id() == null || node.hostId() == null)
+                throw new IllegalArgumentException("Invalid storage node identity");
+            validateUrl(node.url());
+        }
+        this.nodes = List.copyOf(nodes);
+        this.token = token;
+        this.repairToken = repairToken;
+    }
+
+    int count() { return nodes.size(); }
+    List<Node> nodes() { return nodes; }
+    Node node(int index) { return nodes.get(index); }
+    int index(UUID id) {
+        for (int i = 0; i < nodes.size(); i++) if (nodes.get(i).id().equals(id)) return i;
+        return -1;
+    }
+    UUID faultDomain(int index, boolean testNodeDomains) {
+        Node node = nodes.get(index);
+        return testNodeDomains ? node.id() : node.hostId();
+    }
+
+    static NodeIdentity probe(URI url, String token) throws IOException {
+        validateUrl(url);
+        if (token == null || token.length() < 32) throw new IllegalArgumentException("Invalid cluster token");
+        HttpRequest request = HttpRequest.newBuilder(url.resolve("/identity"))
+            .timeout(Duration.ofSeconds(2)).header("X-Cluster-Token", token).GET().build();
+        try {
+            HttpResponse<InputStream> response = IDENTITY_HTTP.send(request, HttpResponse.BodyHandlers.ofInputStream());
+            try (InputStream body = response.body()) {
+                if (response.statusCode() != 200) throw new IOException("Node identity request failed: " + response.statusCode());
+                byte[] bytes = body.readNBytes(128);
+                if (bytes.length == 128) throw new IOException("Node identity response is too large");
+                String[] parts = new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).trim().split(" ", -1);
+                if (parts.length != 2) throw new IOException("Invalid node identity response");
+                return new NodeIdentity(UUID.fromString(parts[0]), UUID.fromString(parts[1]));
+            }
+        } catch (InterruptedException error) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted during node identity request", error);
+        } catch (IllegalArgumentException error) {
+            throw new IOException("Invalid node identity response", error);
+        }
+    }
+
+    static void validateUrl(URI url) {
+        if (url == null || !"http".equals(url.getScheme()) || url.getHost() == null ||
+            url.getPort() < 1 || url.getRawUserInfo() != null ||
+            (url.getRawPath() != null && !url.getRawPath().isEmpty()) ||
+            url.getRawQuery() != null || url.getRawFragment() != null)
+            throw new IllegalArgumentException("Invalid private storage node URL");
+    }
+
+    boolean availableHostsAtLeast(int required, boolean testNodeDomains) {
+        Set<UUID> healthy = new HashSet<>();
+        for (int i = 0; i < nodes.size(); i++) {
+            Node node = nodes.get(i);
+            try {
+                NodeIdentity actual = probe(node.url(), token);
+                if (actual.nodeId().equals(node.id()) && actual.hostId().equals(node.hostId()))
+                    healthy.add(faultDomain(i, testNodeDomains));
+                if (healthy.size() >= required) return true;
+            } catch (IOException error) { }
+        }
+        return false;
+    }
+
+    void put(int index, UUID id, byte[] data, byte[] sha256) throws IOException {
+        put(index, id, data, sha256, false);
+    }
+
+    void repair(int index, UUID id, byte[] data, byte[] sha256) throws IOException {
+        put(index, id, data, sha256, true);
+    }
+
+    private void put(int index, UUID id, byte[] data, byte[] sha256, boolean repair) throws IOException {
+        Node node = nodes.get(index);
+        HttpRequest.Builder builder = HttpRequest.newBuilder(node.url().resolve("/segments/" + id))
+            .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token)
+            .header("X-Cluster-Expected-Node", node.id().toString())
+            .header("X-Cluster-Sha256", HexFormat.of().formatHex(sha256));
+        if (repair) {
+            if (repairToken == null || repairToken.length() < 32)
+                throw new IOException("Repair authority is not available to this process");
+            builder.header("X-Cluster-Repair", "true").header("X-Cluster-Repair-Token", repairToken);
+        }
+        HttpResponse<Void> response = send(builder.PUT(HttpRequest.BodyPublishers.ofByteArray(data)).build(),
+            HttpResponse.BodyHandlers.discarding());
+        if (response.statusCode() != 200) throw new IOException("Node " + node.id() + " rejected segment: " + response.statusCode());
+    }
+
+    byte[] get(int index, UUID id, int length, byte[] sha256) throws IOException {
+        if (length < 1 || length > ClusterNode.MAX_SEGMENT) throw new IOException("Invalid segment length");
+        Node node = nodes.get(index);
+        HttpRequest request = HttpRequest.newBuilder(node.url().resolve("/segments/" + id))
+            .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token)
+            .header("X-Cluster-Expected-Node", node.id().toString()).GET().build();
+        HttpResponse<InputStream> response = send(request, HttpResponse.BodyHandlers.ofInputStream());
+        try (InputStream body = response.body()) {
+            if (response.statusCode() != 200)
+                throw new IOException("Node " + node.id() + " has no verified copy of segment " + id);
+            byte[] bytes = body.readNBytes(length + 1);
+            if (bytes.length != length || !MessageDigest.isEqual(SigV4.hash(bytes), sha256))
+                throw new IOException("Node " + node.id() + " has no verified copy of segment " + id);
+            return bytes;
+        }
+    }
+
+    private <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> handler) throws IOException {
+        try { return http.send(request, handler); }
+        catch (InterruptedException error) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted during node request", error);
+        }
+    }
+}

src/cloud/lunarsky/store/NodeIdentity.java

@@ -0,0 +1,42 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.UUID;
+
+record NodeIdentity(UUID nodeId, UUID hostId) {
+    static NodeIdentity open(Path root, UUID expectedHost) throws IOException {
+        Path file = root.resolve("node-identity");
+        if (Files.exists(file)) {
+            List<String> lines = Files.readAllLines(file, StandardCharsets.UTF_8);
+            if (lines.size() != 2) throw new IOException("Invalid node identity file");
+            try {
+                NodeIdentity identity = new NodeIdentity(UUID.fromString(lines.get(0)), UUID.fromString(lines.get(1)));
+                if (!identity.hostId().equals(expectedHost))
+                    throw new IOException("Node volume belongs to a different storage host");
+                return identity;
+            } catch (IllegalArgumentException error) {
+                throw new IOException("Invalid node identity file", error);
+            }
+        }
+        NodeIdentity identity = new NodeIdentity(UUID.randomUUID(), expectedHost);
+        Path temporary = Files.createTempFile(root, ".node-identity-", ".pending");
+        try {
+            Files.writeString(temporary, identity.nodeId() + "\n" + identity.hostId() + "\n", StandardCharsets.UTF_8);
+            try (FileChannel channel = FileChannel.open(temporary, StandardOpenOption.WRITE)) {
+                channel.force(true);
+            }
+            Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE);
+            DiskStore.syncDirectory(root);
+            return identity;
+        } finally {
+            Files.deleteIfExists(temporary);
+        }
+    }
+}

src/cloud/lunarsky/store/NodeRegistry.java

@@ -0,0 +1,135 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+final class NodeRegistry {
+    private NodeRegistry() {}
+
+    static NodeClient.Node join(Connection connection, URI url, UUID expectedHost, String token) throws IOException {
+        NodeIdentity identity = NodeClient.probe(url, token);
+        if (!identity.hostId().equals(expectedHost))
+            throw new IOException("The node reported a different physical host ID");
+        try {
+            connection.setAutoCommit(false);
+            try (Statement statement = connection.createStatement()) {
+                statement.execute("SELECT pg_advisory_xact_lock(6834071092781)");
+            }
+            try (PreparedStatement query = connection.prepareStatement(
+                    "SELECT node_id, host_id, endpoint FROM cluster_nodes WHERE node_id=? OR endpoint=?")) {
+                query.setObject(1, identity.nodeId());
+                query.setString(2, url.toString());
+                try (ResultSet result = query.executeQuery()) {
+                    if (result.next()) {
+                        if (!identity.nodeId().equals(result.getObject(1)) ||
+                            !identity.hostId().equals(result.getObject(2)) || !url.toString().equals(result.getString(3)))
+                            throw new IOException("Node ID, host ID, or endpoint conflicts with an existing registration");
+                        if (result.next()) throw new IOException("Conflicting node registrations");
+                        connection.commit();
+                        return new NodeClient.Node(identity.nodeId(), identity.hostId(), url);
+                    }
+                }
+            }
+            try (PreparedStatement insert = connection.prepareStatement(
+                    "INSERT INTO cluster_nodes (node_id, host_id, endpoint, state) VALUES (?, ?, ?, 'active')")) {
+                insert.setObject(1, identity.nodeId());
+                insert.setObject(2, identity.hostId());
+                insert.setString(3, url.toString());
+                insert.executeUpdate();
+            }
+            connection.commit();
+            return new NodeClient.Node(identity.nodeId(), identity.hostId(), url);
+        } catch (SQLException | IOException error) {
+            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
+            if (error instanceof IOException io) throw io;
+            throw new IOException("Node registration failed", error);
+        } finally {
+            try { connection.setAutoCommit(true); }
+            catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); }
+        }
+    }
+
+    static NodeClient load(Connection connection, List<URI> urls, String token, String repairToken) throws IOException {
+        if (urls.size() < 2 || urls.stream().distinct().count() != urls.size())
+            throw new IllegalArgumentException("At least two distinct node URLs are required");
+        try {
+            connection.setAutoCommit(false);
+            try (Statement statement = connection.createStatement()) {
+                statement.execute("SELECT pg_advisory_xact_lock(6834071092781)");
+            }
+            Map<String, NodeClient.Node> stored = new HashMap<>();
+            try (Statement statement = connection.createStatement();
+                 ResultSet result = statement.executeQuery("SELECT node_id, host_id, endpoint FROM cluster_nodes WHERE state <> 'retired'")) {
+                while (result.next()) {
+                    URI url = URI.create(result.getString(3));
+                    stored.put(url.toString(), new NodeClient.Node((UUID) result.getObject(1),
+                        (UUID) result.getObject(2), url));
+                }
+            }
+            if (stored.isEmpty()) {
+                try (Statement statement = connection.createStatement();
+                     ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_segments)")) {
+                    result.next();
+                    if (result.getBoolean(1)) throw new IOException("Existing segments have no registered node identities");
+                }
+                for (URI url : urls) {
+                    NodeIdentity identity = NodeClient.probe(url, token);
+                    try (PreparedStatement insert = connection.prepareStatement(
+                            "INSERT INTO cluster_nodes (node_id, host_id, endpoint, state) VALUES (?, ?, ?, 'active')")) {
+                        insert.setObject(1, identity.nodeId());
+                        insert.setObject(2, identity.hostId());
+                        insert.setString(3, url.toString());
+                        insert.executeUpdate();
+                    }
+                    stored.put(url.toString(), new NodeClient.Node(identity.nodeId(), identity.hostId(), url));
+                }
+            }
+            List<NodeClient.Node> configured = new ArrayList<>();
+            Set<UUID> configuredIds = new HashSet<>();
+            for (URI url : urls) {
+                NodeClient.Node node = stored.get(url.toString());
+                if (node == null) throw new IOException("Unregistered storage node URL: " + url);
+                NodeIdentity actual = null;
+                try {
+                    actual = NodeClient.probe(url, token);
+                } catch (IOException offline) { }
+                if (actual != null && (!actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())))
+                    throw new IOException("Storage node identity changed at " + url);
+                configured.add(node);
+                configuredIds.add(node.id());
+            }
+            try (Statement statement = connection.createStatement();
+                 ResultSet result = statement.executeQuery(
+                     "SELECT DISTINCT unnest(s.replica_ids) FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation")) {
+                while (result.next()) {
+                    UUID id = (UUID) result.getObject(1);
+                    if (!configuredIds.contains(id))
+                        throw new IOException("A live segment refers to a node missing from CLUSTER_NODES: " + id);
+                }
+            }
+            NodeClient nodes = new NodeClient(configured, token, repairToken);
+            connection.commit();
+            return nodes;
+        } catch (SQLException | IOException | RuntimeException error) {
+            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
+            if (error instanceof IOException io) throw io;
+            if (error instanceof SQLException sql) throw new IOException("Node registry check failed", sql);
+            throw (RuntimeException) error;
+        } finally {
+            try { connection.setAutoCommit(true); }
+            catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); }
+        }
+    }
+}

src/cloud/lunarsky/store/ObjectStorage.java

@@ -0,0 +1,26 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+/** Storage operations shared by the local and cluster gateways. */
+interface ObjectStorage extends AutoCloseable {
+    record Metadata(long length, long modified, String etag, byte[] sha256,
+                    String bucket, String key, String contentType) {}
+    record OpenObject(Metadata metadata, InputStream stream) implements AutoCloseable {
+        public void close() throws IOException { stream.close(); }
+    }
+    record ListedObject(String key, Metadata metadata) {}
+    record ListPage(List<ListedObject> objects, List<String> prefixes, String nextKey, boolean truncated) {
+        int keyCount() { return objects.size() + prefixes.size(); }
+    }
+
+    Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
+                 String checksum, boolean createOnly, String contentType) throws IOException;
+    OpenObject open(String bucket, String key) throws IOException;
+    void delete(String bucket, String key) throws IOException;
+    ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) throws IOException;
+    default boolean ready() { return true; }
+    void close() throws IOException;
+}

src/cloud/lunarsky/store/PlacementPolicy.java

@@ -0,0 +1,38 @@
+package cloud.lunarsky.store;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+final class PlacementPolicy {
+    private PlacementPolicy() {}
+
+    static List<Integer> candidates(UUID segmentId, NodeClient nodes, boolean testNodeDomains) {
+        Map<UUID, List<Integer>> byHost = new HashMap<>();
+        for (int i = 0; i < nodes.count(); i++)
+            byHost.computeIfAbsent(nodes.faultDomain(i, testNodeDomains), ignored -> new ArrayList<>()).add(i);
+        List<UUID> hosts = new ArrayList<>(byHost.keySet());
+        hosts.sort(Comparator.comparingLong((UUID host) -> score(segmentId, host)).reversed());
+        int longest = 0;
+        for (List<Integer> group : byHost.values()) {
+            group.sort(Comparator.comparingLong((Integer index) -> score(segmentId, nodes.node(index).id())).reversed());
+            longest = Math.max(longest, group.size());
+        }
+        List<Integer> order = new ArrayList<>(nodes.count());
+        for (int round = 0; round < longest; round++)
+            for (UUID host : hosts)
+                if (round < byHost.get(host).size()) order.add(byHost.get(host).get(round));
+        return order;
+    }
+
+    private static long score(UUID segment, UUID candidate) {
+        ByteBuffer bytes = ByteBuffer.allocate(32);
+        bytes.putLong(segment.getMostSignificantBits()).putLong(segment.getLeastSignificantBits());
+        bytes.putLong(candidate.getMostSignificantBits()).putLong(candidate.getLeastSignificantBits());
+        return ByteBuffer.wrap(SigV4.hash(bytes.array())).getLong();
+    }
+}

src/cloud/lunarsky/store/SchemaMigrator.java

@@ -0,0 +1,71 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+final class SchemaMigrator {
+    private SchemaMigrator() {}
+
+    static int prepare(Connection connection, String bucket) throws IOException {
+        try {
+            connection.setAutoCommit(false);
+            try (Statement statement = connection.createStatement()) {
+                statement.execute("SELECT pg_advisory_xact_lock(6834071092781)");
+                statement.execute("CREATE TABLE IF NOT EXISTS cluster_schema_migrations (version integer PRIMARY KEY)");
+                int version;
+                try (ResultSet result = statement.executeQuery("SELECT COALESCE(MAX(version), 0) FROM cluster_schema_migrations")) {
+                    result.next();
+                    version = result.getInt(1);
+                }
+                if (version > 2) throw new IOException("Metadata schema is newer than this ObjectStore build");
+                if (version < 1) {
+                    statement.execute("CREATE TABLE IF NOT EXISTS cluster_usage (bucket text PRIMARY KEY, used_bytes bigint NOT NULL CHECK (used_bytes >= 0))");
+                    statement.execute("CREATE TABLE IF NOT EXISTS cluster_objects (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, length bigint NOT NULL, modified bigint NOT NULL, etag text NOT NULL, sha256 bytea NOT NULL, content_type text NOT NULL, PRIMARY KEY (bucket, object_key))");
+                    statement.execute("CREATE TABLE IF NOT EXISTS cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))");
+                    statement.execute("CREATE TABLE IF NOT EXISTS cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (1)");
+                }
+                if (version < 2) {
+                    statement.execute("ALTER TABLE cluster_segments ADD COLUMN IF NOT EXISTS replica_ids uuid[]");
+                    statement.execute("ALTER TABLE cluster_segments ADD COLUMN IF NOT EXISTS placement_version bigint NOT NULL DEFAULT 0");
+                    statement.execute("ALTER TABLE cluster_segments ADD CONSTRAINT cluster_replica_ids_required CHECK (replica_ids IS NOT NULL) NOT VALID");
+                    statement.execute("CREATE TABLE IF NOT EXISTS cluster_nodes (node_id uuid PRIMARY KEY, host_id uuid NOT NULL, endpoint text NOT NULL UNIQUE, legacy_index integer UNIQUE, state text NOT NULL CHECK (state IN ('joining','active','draining','offline','retired')))");
+                    statement.execute("CREATE TABLE IF NOT EXISTS cluster_format (singleton integer PRIMARY KEY CHECK (singleton=1), version integer NOT NULL)");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (2)");
+                }
+                statement.execute("INSERT INTO cluster_format SELECT 1, CASE WHEN EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL) THEN 1 ELSE 2 END WHERE NOT EXISTS (SELECT 1 FROM cluster_format)");
+            }
+            try (PreparedStatement insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, 0) ON CONFLICT DO NOTHING")) {
+                insert.setString(1, bucket);
+                insert.executeUpdate();
+            }
+            int format;
+            try (Statement statement = connection.createStatement();
+                 ResultSet result = statement.executeQuery("SELECT version FROM cluster_format WHERE singleton=1")) {
+                if (!result.next()) throw new IOException("Missing cluster format marker");
+                format = result.getInt(1);
+            }
+            if (format < 1 || format > 2) throw new IOException("Unsupported cluster data format " + format);
+            if (format == 2) {
+                try (Statement statement = connection.createStatement();
+                     ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL)")) {
+                    result.next();
+                    if (result.getBoolean(1)) throw new IOException("Cluster format has unmigrated segment replicas");
+                }
+            }
+            connection.commit();
+            return format;
+        } catch (SQLException | IOException error) {
+            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
+            if (error instanceof IOException io) throw io;
+            throw new IOException("Metadata schema migration failed", error);
+        } finally {
+            try { connection.setAutoCommit(true); }
+            catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); }
+        }
+    }
+}

src/cloud/lunarsky/store/StoreException.java

@@ -4,7 +4,10 @@ final class StoreException extends RuntimeException {
     final int status;
     final String code;
     StoreException(int status, String code, String message) {
-        super(message);
+        this(status, code, message, null);
+    }
+    StoreException(int status, String code, String message, Throwable cause) {
+        super(message, cause);
         this.status = status;
         this.code = code;
     }

src/cloud/lunarsky/store/UnavailableMultipart.java

@@ -0,0 +1,17 @@
+package cloud.lunarsky.store;
+
+import java.io.InputStream;
+import java.util.List;
+
+final class UnavailableMultipart implements MultipartStorage {
+    private StoreException unavailable() {
+        return new StoreException(501, "NotImplemented", "Multipart uploads are unavailable in the local cluster prototype");
+    }
+    @Override public String create(String bucket, String key, String contentType) { throw unavailable(); }
+    @Override public String putPart(String id, String bucket, String key, int number, InputStream input,
+                                    long length, String expectedHash, String checksum) { throw unavailable(); }
+    @Override public ObjectStorage.Metadata complete(String id, String bucket, String key, List<Part> parts) { throw unavailable(); }
+    @Override public void abort(String id, String bucket, String key) { throw unavailable(); }
+    @Override public int activeUploads() { return 0; }
+    @Override public long stagedBytes() { return 0; }
+}

src/cloud/lunarsky/store/Version.java

@@ -0,0 +1,7 @@
+package cloud.lunarsky.store;
+
+final class Version {
+    static final String VALUE = "0.0.2";
+
+    private Version() {}
+}

test/cloud/lunarsky/store/CliTest.java

@@ -0,0 +1,54 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.PrintStream;
+import java.io.RandomAccessFile;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+
+public final class CliTest {
+    public static void main(String[] args) throws Exception {
+        Path root = Files.createTempDirectory("store-cli-test-");
+        try {
+            var versionOutput = new ByteArrayOutputStream();
+            if (Cli.run(new String[]{"version"}, root, new PrintStream(versionOutput), System.err) != 0 ||
+                !versionOutput.toString().contains(Version.VALUE)) throw new AssertionError("CLI version");
+            byte[] payload = "verified".getBytes(StandardCharsets.UTF_8);
+            try (var store = new DiskStore(root, 100, 1000)) {
+                store.put("objects", "example", new ByteArrayInputStream(payload), payload.length,
+                    SigV4.hex(SigV4.hash(payload)), null, false, "text/plain");
+                var output = new ByteArrayOutputStream();
+                int status = Cli.run(new String[]{"status"}, root, new PrintStream(output), System.err);
+                if (status != 0 || !output.toString().contains("objects=1") ||
+                    !output.toString().contains("payload_bytes=8")) throw new AssertionError("CLI status");
+                output.reset();
+                status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), System.err);
+                if (status != 0 || !output.toString().contains("verified_objects=1"))
+                    throw new AssertionError("CLI verification");
+            }
+            String id = SigV4.hex(SigV4.hash("objects/example".getBytes(StandardCharsets.UTF_8)));
+            Path file = root.resolve("objects").resolve(id.substring(0, 2)).resolve(id);
+            int header;
+            try (var input = new DataInputStream(Files.newInputStream(file))) {
+                header = DiskStore.readRecord(input).headerLength();
+            }
+            try (var bytes = new RandomAccessFile(file.toFile(), "rw")) {
+                bytes.seek(header);
+                bytes.write('X');
+            }
+            var output = new ByteArrayOutputStream();
+            int status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), new PrintStream(output));
+            if (status != 1 || !output.toString().contains("Object checksum mismatch"))
+                throw new AssertionError("CLI missed corrupted payload");
+            System.out.println("CLI tests passed: version, live status, verification, corruption exit code");
+        } finally {
+            try (var paths = Files.walk(root)) {
+                for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path);
+            }
+        }
+    }
+}

test/cloud/lunarsky/store/ClusterIntegrationTest.java

@@ -0,0 +1,144 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Map;
+
+public final class ClusterIntegrationTest {
+    private static final String KEY = "cluster-test/survivor";
+    public static void main(String[] args) throws Exception {
+        Map<String, String> env = System.getenv();
+        String[] urls = env.get("CLUSTER_NODES").split(",");
+        try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"),
+                env.get("POSTGRES_PASSWORD"), env.get("S3_BUCKET"),
+                Arrays.stream(urls).map(URI::create).toList(), env.get("CLUSTER_TOKEN"), null,
+                134217728, 2147483648L,
+                !"same-host".equals(args[0]) && "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS")))) {
+            String bucket = env.get("S3_BUCKET");
+            byte[] stable = "acknowledged object survives node loss".getBytes(StandardCharsets.UTF_8);
+            switch (args[0]) {
+                case "basic" -> {
+                    byte[] large = new byte[ClusterNode.MAX_SEGMENT + 37];
+                    for (int i = 0; i < large.length; i++) large[i] = (byte) (i * 31);
+                    String largeKey = "cluster-test/large";
+                    put(store, bucket, largeKey, large, false);
+                    try (var opened = store.open(bucket, largeKey)) {
+                        require(opened.metadata().length() == large.length, "Wrong object length");
+                        require(Arrays.equals(large, opened.stream().readAllBytes()), "Multi-segment read mismatch");
+                    }
+                    require(store.list(bucket, "cluster-test/", "", 10, null).keyCount() >= 1, "Listing missed object");
+                    try {
+                        put(store, bucket, largeKey, large, true);
+                        throw new AssertionError("Create-only overwrite succeeded");
+                    } catch (StoreException error) { require(error.status == 412, "Wrong create-only status"); }
+                    put(store, bucket, largeKey, "replacement".getBytes(StandardCharsets.UTF_8), false);
+                    try (var opened = store.open(bucket, largeKey)) {
+                        require("replacement".equals(new String(opened.stream().readAllBytes(), StandardCharsets.UTF_8)),
+                            "Overwrite was not visible");
+                    }
+                    store.delete(bucket, largeKey);
+                    try { store.open(bucket, largeKey); throw new AssertionError("Deleted object remained visible"); }
+                    catch (StoreException error) { require(error.status == 404, "Wrong missing-object status"); }
+                    put(store, bucket, KEY, stable, false);
+                    require(store.ready(), "Healthy cluster is not ready");
+                    System.out.println("Cluster basic test passed");
+                }
+                case "degraded" -> {
+                    require(store.ready(), "Two available nodes should be ready");
+                    try (var opened = store.open(bucket, KEY)) {
+                        require(Arrays.equals(stable, opened.stream().readAllBytes()), "Acknowledged object was lost");
+                    }
+                    byte[] value = "written with one node down".getBytes(StandardCharsets.UTF_8);
+                    put(store, bucket, "cluster-test/degraded", value, false);
+                    try (var opened = store.open(bucket, "cluster-test/degraded")) {
+                        require(Arrays.equals(value, opened.stream().readAllBytes()), "Degraded write was not readable");
+                    }
+                    System.out.println("Cluster degraded test passed");
+                }
+                case "quorum-lost" -> {
+                    require(!store.ready(), "One available node must not be ready");
+                    try {
+                        put(store, bucket, "cluster-test/rejected", new byte[]{1}, false);
+                        throw new AssertionError("Write succeeded with only one node");
+                    } catch (StoreException error) { require(error.status == 503, "Wrong unavailable status"); }
+                    try { store.open(bucket, "cluster-test/rejected"); throw new AssertionError("Failed write became visible"); }
+                    catch (StoreException error) { require(error.status == 404, "Partial object became visible"); }
+                    System.out.println("Cluster quorum-loss test passed");
+                }
+                case "recovered" -> {
+                    require(store.ready(), "Restarted cluster is not ready");
+                    try (var opened = store.open(bucket, KEY)) {
+                        require(Arrays.equals(stable, opened.stream().readAllBytes()), "Object lost across restart");
+                    }
+                    System.out.println("Cluster recovery test passed");
+                }
+                case "concurrent" -> {
+                    String key = "cluster-test/concurrent";
+                    byte[] first = "concurrent-first".getBytes(StandardCharsets.UTF_8);
+                    byte[] second = "concurrent-second".getBytes(StandardCharsets.UTF_8);
+                    var start = new java.util.concurrent.CountDownLatch(1);
+                    var executor = java.util.concurrent.Executors.newFixedThreadPool(2);
+                    try {
+                        var a = executor.submit(() -> {
+                            start.await(); put(store, bucket, key, first, false); return null;
+                        });
+                        var b = executor.submit(() -> {
+                            start.await(); put(store, bucket, key, second, false); return null;
+                        });
+                        start.countDown();
+                        a.get(); b.get();
+                        try (var opened = store.open(bucket, key)) {
+                            byte[] actual = opened.stream().readAllBytes();
+                            require(Arrays.equals(actual, first) || Arrays.equals(actual, second),
+                                "Concurrent PUT produced a partial object");
+                        }
+                        require(store.list(bucket, key, "", 10, null).keyCount() == 1,
+                            "Concurrent PUT produced duplicate key entries");
+                    } finally { executor.shutdownNow(); }
+                    System.out.println("Cluster concurrent overwrite test passed");
+                }
+                case "same-host" -> {
+                    require(!store.ready(), "Containers on one physical host must not form a storage quorum");
+                    try {
+                        put(store, bucket, "cluster-test/same-host-rejected", new byte[]{1}, false);
+                        throw new AssertionError("Write succeeded without two physical storage hosts");
+                    } catch (StoreException error) { require(error.status == 503, "Wrong same-host rejection status"); }
+                    System.out.println("Same-host replicas correctly fail the physical-host quorum");
+                }
+                case "joined" -> {
+                    require(urls.length == 4, "Expansion test requires four registered nodes");
+                    var newNode = NodeClient.probe(URI.create(urls[3]), env.get("CLUSTER_TOKEN"));
+                    for (int i = 0; i < 32; i++) {
+                        String key = "cluster-test/expanded-" + i;
+                        byte[] value = ("expanded object " + i).getBytes(StandardCharsets.UTF_8);
+                        put(store, bucket, key, value, false);
+                        try (var opened = store.open(bucket, key)) {
+                            require(Arrays.equals(value, opened.stream().readAllBytes()), "Expanded object was not readable");
+                        }
+                    }
+                    try (var connection = java.sql.DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                            env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
+                         var query = connection.prepareStatement(
+                             "SELECT count(*) FROM cluster_segments WHERE ? = ANY(replica_ids)")) {
+                        query.setObject(1, newNode.nodeId());
+                        try (var result = query.executeQuery()) {
+                            result.next();
+                            require(result.getLong(1) > 0, "Joined node received no new segments");
+                        }
+                    }
+                    System.out.println("Joined node accepted new placements while previous objects stayed readable");
+                }
+                default -> throw new IllegalArgumentException("Unknown test phase");
+            }
+        }
+    }
+    private static void put(ClusterStore store, String bucket, String key, byte[] data, boolean createOnly) throws Exception {
+        store.put(bucket, key, new ByteArrayInputStream(data), data.length, SigV4.hex(SigV4.hash(data)),
+            null, createOnly, "application/octet-stream");
+    }
+    private static void require(boolean condition, String message) {
+        if (!condition) throw new AssertionError(message);
+    }
+}

test/cloud/lunarsky/store/ClusterMigrationTest.java

@@ -0,0 +1,89 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.sql.DriverManager;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public final class ClusterMigrationTest {
+    private static final String KEY = "migration/legacy-object";
+    private static final byte[] DATA = "legacy object survives stable-node migration".getBytes(StandardCharsets.UTF_8);
+
+    public static void main(String[] args) throws Exception {
+        Map<String, String> env = System.getenv();
+        List<URI> urls = Arrays.stream(env.get("CLUSTER_NODES").split(","))
+            .map(URI::create).toList();
+        String token = env.get("CLUSTER_TOKEN");
+        List<NodeClient.Node> addresses = new ArrayList<>();
+        for (URI url : urls) {
+            NodeIdentity identity = NodeClient.probe(url, token);
+            addresses.add(new NodeClient.Node(identity.nodeId(), identity.hostId(), url));
+        }
+        NodeClient nodes = new NodeClient(addresses, token, null);
+        String bucket = env.get("S3_BUCKET");
+        if (args[0].equals("create")) {
+            UUID segment = UUID.randomUUID();
+            byte[] hash = SigV4.hash(DATA);
+            for (int i = 0; i < nodes.count(); i++) nodes.put(i, segment, DATA, hash);
+            try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                    env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
+                 var statement = connection.createStatement()) {
+                statement.execute("CREATE TABLE cluster_usage (bucket text PRIMARY KEY, used_bytes bigint NOT NULL)");
+                statement.execute("CREATE TABLE cluster_objects (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, length bigint NOT NULL, modified bigint NOT NULL, etag text NOT NULL, sha256 bytea NOT NULL, content_type text NOT NULL, PRIMARY KEY (bucket, object_key))");
+                statement.execute("CREATE TABLE cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))");
+                statement.execute("CREATE TABLE cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))");
+                try (var insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, ?)")) {
+                    insert.setString(1, bucket); insert.setLong(2, DATA.length); insert.executeUpdate();
+                }
+                UUID generation = UUID.randomUUID();
+                try (var insert = connection.prepareStatement("INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) {
+                    insert.setString(1, bucket); insert.setString(2, KEY); insert.setObject(3, generation);
+                    insert.setLong(4, DATA.length); insert.setLong(5, System.currentTimeMillis());
+                    insert.setString(6, HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(DATA)));
+                    insert.setBytes(7, hash); insert.setString(8, "text/plain"); insert.executeUpdate();
+                }
+                try (var insert = connection.prepareStatement("INSERT INTO cluster_segments VALUES (?, 0, ?, ?, ?, '0,1,2')")) {
+                    insert.setObject(1, generation); insert.setObject(2, segment);
+                    insert.setInt(3, DATA.length); insert.setBytes(4, hash); insert.executeUpdate();
+                }
+            }
+            System.out.println("Legacy cluster fixture created");
+            return;
+        }
+        if (!args[0].equals("verify")) throw new IllegalArgumentException("Use create or verify");
+        List<URI> reordered = new ArrayList<>(urls);
+        java.util.Collections.reverse(reordered);
+        try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"),
+                env.get("POSTGRES_PASSWORD"), bucket, reordered, token, null,
+                134217728, 2147483648L, true)) {
+            try (var opened = store.open(bucket, KEY)) {
+                if (!Arrays.equals(DATA, opened.stream().readAllBytes()))
+                    throw new AssertionError("Migrated object changed after node URL reorder");
+            }
+            store.put(bucket, "migration/new-object", new ByteArrayInputStream(DATA), DATA.length,
+                SigV4.hex(SigV4.hash(DATA)), null, false, "text/plain");
+        }
+        UUID segment;
+        try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
+             var statement = connection.createStatement();
+             var result = statement.executeQuery("SELECT segment_id FROM cluster_segments WHERE replicas='0,1,2' LIMIT 1")) {
+            if (!result.next()) throw new AssertionError("Legacy segment missing after migration");
+            segment = (UUID) result.getObject(1);
+        }
+        NodeClient wrong = new NodeClient(List.of(new NodeClient.Node(addresses.get(0).id(),
+            addresses.get(0).hostId(), addresses.get(1).url())), token, null);
+        try {
+            wrong.get(0, segment, DATA.length, SigV4.hash(DATA));
+            throw new AssertionError("Node swap was not rejected");
+        } catch (java.io.IOException expected) { }
+        System.out.println("Stable node migration, reordered gateway config, and wrong-node rejection passed");
+    }
+}

test/cloud/lunarsky/store/ClusterNodeTest.java

@@ -0,0 +1,124 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.UUID;
+
+public final class ClusterNodeTest {
+    public static void main(String[] args) throws Exception {
+        Path root = Files.createTempDirectory("objectstore-node-");
+        String token = "local-cluster-test-token-0123456789";
+        String repairToken = "local-repair-test-token-0123456789";
+        UUID id = UUID.randomUUID();
+        UUID hostId = UUID.randomUUID();
+        UUID nodeId;
+        byte[] value = "a durable segment".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        try (ClusterNode node = new ClusterNode(root, token, repairToken, hostId)) {
+            try {
+                new ClusterNode(root, token, repairToken, hostId);
+                throw new AssertionError("Second writer opened a locked node directory");
+            } catch (IOException expected) { }
+            HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+            server.createContext("/", node::handle);
+            server.start();
+            try {
+                URI uri = URI.create("http://127.0.0.1:" + server.getAddress().getPort());
+                NodeIdentity identity = NodeClient.probe(uri, token);
+                nodeId = identity.nodeId();
+                require(identity.hostId().equals(hostId), "Node reported the wrong storage host");
+                NodeClient client = new NodeClient(List.of(new NodeClient.Node(identity.nodeId(), hostId, uri)),
+                    token, repairToken);
+                client.put(0, id, value, SigV4.hash(value));
+                client.put(0, id, value, SigV4.hash(value));
+                require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))), "Roundtrip failed");
+                var request = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).timeout(Duration.ofSeconds(3))
+                    .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString())
+                    .header("X-Cluster-Sha256", SigV4.hex(SigV4.hash(value)))
+                    .PUT(HttpRequest.BodyPublishers.ofString("wrong")).build();
+                var response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
+                require(response.statusCode() == 400, "Wrong checksum accepted");
+                var unauthenticated = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).GET().build();
+                require(HttpClient.newHttpClient().send(unauthenticated, HttpResponse.BodyHandlers.ofString())
+                    .statusCode() == 403, "Unauthenticated read accepted");
+                var wrongNode = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).timeout(Duration.ofSeconds(3))
+                    .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", UUID.randomUUID().toString())
+                    .GET().build();
+                require(HttpClient.newHttpClient().send(wrongNode, HttpResponse.BodyHandlers.ofString())
+                    .statusCode() == 409, "Wrong-node request was accepted");
+                Path segment = root.resolve("segments").resolve(id.toString().substring(0, 2)).resolve(id.toString());
+                Files.writeString(segment, "corrupted");
+                try {
+                    client.get(0, id, value.length, SigV4.hash(value));
+                    throw new AssertionError("Corrupted replica passed verification");
+                } catch (IOException expected) { }
+                NodeClient gateway = new NodeClient(List.of(new NodeClient.Node(identity.nodeId(), hostId, uri)),
+                    token, null);
+                try {
+                    gateway.repair(0, id, value, SigV4.hash(value));
+                    throw new AssertionError("Gateway was allowed to repair a replica");
+                } catch (IOException expected) { }
+                var forgedRepair = HttpRequest.newBuilder(uri.resolve("/segments/" + id))
+                    .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString())
+                    .header("X-Cluster-Repair", "true").header("X-Cluster-Sha256", SigV4.hex(SigV4.hash(value)))
+                    .PUT(HttpRequest.BodyPublishers.ofByteArray(value)).build();
+                require(HttpClient.newHttpClient().send(forgedRepair, HttpResponse.BodyHandlers.discarding())
+                    .statusCode() == 403, "Shared node token was allowed to repair a replica");
+                client.repair(0, id, value, SigV4.hash(value));
+                require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))),
+                    "Repair did not restore the original bytes");
+            } finally { server.stop(0); }
+        }
+        Path pending = root.resolve("pending").resolve("unfinished.part");
+        Files.writeString(pending, "unfinished");
+        try (ClusterNode restarted = new ClusterNode(root, token, repairToken, hostId)) {
+            require(!Files.exists(pending), "Incomplete segment survived restart");
+            Path segment = root.resolve("segments").resolve(id.toString().substring(0, 2)).resolve(id.toString());
+            require(java.util.Arrays.equals(value, Files.readAllBytes(segment)), "Committed segment did not survive restart");
+        }
+        require(NodeIdentity.open(root, hostId).nodeId().equals(nodeId),
+            "Node identity changed after restart");
+        try {
+            new ClusterNode(root, token, repairToken, UUID.randomUUID());
+            throw new AssertionError("Node volume accepted a changed host identity");
+        } catch (IOException expected) { }
+        HttpServer oversized = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        oversized.createContext("/identity", exchange -> {
+            byte[] bytes = new byte[1024];
+            exchange.sendResponseHeaders(200, 0);
+            exchange.getResponseBody().write(bytes);
+            exchange.close();
+        });
+        oversized.createContext("/segments/", exchange -> {
+            byte[] bytes = new byte[1024];
+            exchange.sendResponseHeaders(200, 0);
+            exchange.getResponseBody().write(bytes);
+            exchange.close();
+        });
+        oversized.start();
+        try {
+            URI uri = URI.create("http://127.0.0.1:" + oversized.getAddress().getPort());
+            try {
+                NodeClient.probe(uri, token);
+                throw new AssertionError("Oversized identity response was accepted");
+            } catch (IOException expected) { }
+            NodeClient client = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, uri)), token, null);
+            try {
+                client.get(0, id, value.length, SigV4.hash(value));
+                throw new AssertionError("Oversized segment response was accepted");
+            } catch (IOException expected) { }
+        } finally { oversized.stop(0); }
+        System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup");
+    }
+    private static void require(boolean condition, String message) {
+        if (!condition) throw new AssertionError(message);
+    }
+}

test/cloud/lunarsky/store/ConcurrencyTest.java

@@ -0,0 +1,88 @@
+package cloud.lunarsky.store;
+
+import java.io.ByteArrayInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public final class ConcurrencyTest {
+    private static ObjectStorage.Metadata put(DiskStore store, byte[] body) throws Exception {
+        return store.put("test", "shared", new ByteArrayInputStream(body), body.length,
+            SigV4.hex(SigV4.hash(body)), null, false, "application/octet-stream");
+    }
+
+    private static void check(DiskStore store, byte[] first, byte[] second) throws Exception {
+        try (var object = store.open("test", "shared")) {
+            byte[] body = object.stream().readAllBytes();
+            if (!Arrays.equals(body, first) && !Arrays.equals(body, second))
+                throw new AssertionError("Reader observed a partial object");
+            if (object.metadata().length() != body.length ||
+                !Arrays.equals(object.metadata().sha256(), SigV4.hash(body)))
+                throw new AssertionError("Reader observed mismatched metadata");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Path root = Files.createTempDirectory("store-concurrency-test-");
+        byte[] first = new byte[65536];
+        byte[] second = new byte[81920];
+        Arrays.fill(first, (byte) 0x35);
+        Arrays.fill(second, (byte) 0x67);
+        try (var store = new DiskStore(root, 131072, 131072);
+             var workers = Executors.newFixedThreadPool(4)) {
+            put(store, first);
+            var start = new CountDownLatch(1);
+            var done = new AtomicBoolean(false);
+            Future<?> writer = workers.submit(() -> {
+                try {
+                    start.await();
+                    for (int i = 0; i < 100; i++) {
+                        byte[] body = i % 2 == 0 ? second : first;
+                        var saved = put(store, body);
+                        var listed = store.list("test", "", "", 10, null).objects();
+                        if (listed.size() != 1 || !listed.getFirst().metadata().etag().equals(saved.etag()))
+                            throw new AssertionError("Acknowledged write is absent from listing");
+                        check(store, first, second);
+                    }
+                } catch (Exception error) {
+                    throw new RuntimeException(error);
+                } finally {
+                    done.set(true);
+                }
+            });
+            Future<?>[] readers = new Future<?>[3];
+            for (int i = 0; i < readers.length; i++) {
+                readers[i] = workers.submit(() -> {
+                    try {
+                        start.await();
+                        while (!done.get()) check(store, first, second);
+                    } catch (Exception error) {
+                        throw new RuntimeException(error);
+                    }
+                });
+            }
+            start.countDown();
+            writer.get();
+            for (Future<?> reader : readers) reader.get();
+            store.delete("test", "shared");
+            if (!store.list("test", "", "", 10, null).objects().isEmpty())
+                throw new AssertionError("Acknowledged delete is present in listing");
+            try {
+                store.open("test", "shared");
+                throw new AssertionError("Acknowledged delete remained readable");
+            } catch (StoreException expected) {
+                if (expected.status != 404) throw expected;
+            }
+            System.out.println("Concurrent storage tests passed: atomic overwrite, read and listing after acknowledgement, delete");
+        } finally {
+            try (var paths = Files.walk(root)) {
+                for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path);
+            }
+        }
+    }
+}

test/cloud/lunarsky/store/HttpTest.java

@@ -14,6 +14,8 @@ import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.concurrent.Executors;
 
 public final class HttpTest {
@@ -24,26 +26,34 @@ public final class HttpTest {
         DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);
 
     private static HttpRequest signed(String base, String method, String key, byte[] body) {
-        URI uri = URI.create(base + "/objects/" + SigV4.encode(key, true));
+        return signedUri(URI.create(base + "/objects/" + SigV4.encode(key, true)), method, body, Map.of());
+    }
+
+    private static HttpRequest signedUri(URI uri, String method, byte[] body, Map<String, String> extra) {
         String host = uri.getAuthority();
         String date = DATE.format(Instant.now());
         String hash = SigV4.hex(SigV4.hash(body));
-        String names = "host;x-amz-content-sha256;x-amz-date";
-        String canonical = method + "\n" + uri.getRawPath() + "\n\n"
-            + "host:" + host + "\n"
-            + "x-amz-content-sha256:" + hash + "\n"
-            + "x-amz-date:" + date + "\n\n" + names + "\n" + hash;
+        TreeMap<String, String> signed = new TreeMap<>(extra);
+        signed.put("host", host);
+        signed.put("x-amz-content-sha256", hash);
+        signed.put("x-amz-date", date);
+        String names = String.join(";", signed.keySet());
+        StringBuilder canonical = new StringBuilder(method).append('\n').append(uri.getRawPath()).append('\n')
+            .append(SigV4.canonicalQuery(uri.getRawQuery())).append('\n');
+        signed.forEach((name, value) -> canonical.append(name).append(':').append(value).append('\n'));
+        canonical.append('\n').append(names).append('\n').append(hash);
         String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
         String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
-            + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)));
+            + SigV4.hex(SigV4.hash(canonical.toString().getBytes(StandardCharsets.UTF_8)));
         String signature = SigV4.hex(SigV4.hmac(
             SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign));
-        return HttpRequest.newBuilder(uri)
+        HttpRequest.Builder request = HttpRequest.newBuilder(uri)
             .header("x-amz-date", date)
             .header("x-amz-content-sha256", hash)
             .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/"
-                + scope + ",SignedHeaders=" + names + ",Signature=" + signature)
-            .method(method, body.length == 0
+                + scope + ",SignedHeaders=" + names + ",Signature=" + signature);
+        extra.forEach(request::header);
+        return request.method(method, body.length == 0
                 ? HttpRequest.BodyPublishers.noBody()
                 : HttpRequest.BodyPublishers.ofByteArray(body))
             .build();
@@ -77,12 +87,85 @@ public final class HttpTest {
                 .GET().build(), HttpResponse.BodyHandlers.ofByteArray()));
             status(200, client.send(signed(base, "PUT", key, body),
                 HttpResponse.BodyHandlers.ofByteArray()));
+            String other = "folder/stars.txt";
+            status(200, client.send(signedUri(URI.create(base + "/objects/" + other), "PUT",
+                "stars".getBytes(StandardCharsets.UTF_8), Map.of("content-type", "text/plain")),
+                HttpResponse.BodyHandlers.ofByteArray()));
             var get = client.send(signed(base, "GET", key, new byte[0]),
                 HttpResponse.BodyHandlers.ofByteArray());
             status(200, get);
             if (!java.util.Arrays.equals(body, get.body())) throw new AssertionError("GET body mismatch");
             if (!"application/octet-stream".equals(get.headers().firstValue("content-type").orElse("")))
                 throw new AssertionError("Unexpected content type");
+            var typed = client.send(signed(base, "GET", other, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
+            status(200, typed);
+            if (!"text/plain".equals(typed.headers().firstValue("content-type").orElse("")))
+                throw new AssertionError("Stored content type missing");
+            var partial = client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
+                new byte[0], Map.of("range", "bytes=1-3")), HttpResponse.BodyHandlers.ofByteArray());
+            status(206, partial);
+            if (!"tar".equals(new String(partial.body(), StandardCharsets.UTF_8)) ||
+                !"bytes 1-3/5".equals(partial.headers().firstValue("content-range").orElse("")))
+                throw new AssertionError("Range response mismatch");
+            status(416, client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
+                new byte[0], Map.of("range", "bytes=20-30")), HttpResponse.BodyHandlers.ofByteArray()));
+            var listed = client.send(signedUri(URI.create(base + "/objects?list-type=2&prefix=folder%2F"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (listed.statusCode() != 200 || !listed.body().contains("<Key>folder/stars.txt</Key>") ||
+                !listed.body().contains("<Key>folder/moon-☾.txt</Key>"))
+                throw new AssertionError("ListObjectsV2 failed: " + listed.body());
+            var page = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=1"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (page.statusCode() != 200 || !page.body().contains("<IsTruncated>true</IsTruncated>"))
+                throw new AssertionError("List pagination failed: " + page.body());
+            String token = page.body().split("<NextContinuationToken>")[1].split("</NextContinuationToken>")[0];
+            var next = client.send(signedUri(URI.create(base + "/objects?list-type=2&continuation-token=" + token),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (next.statusCode() != 200 || !next.body().contains("<Key>folder/stars.txt</Key>"))
+                throw new AssertionError("List continuation failed: " + next.body());
+            var grouped = client.send(signedUri(URI.create(base + "/objects?list-type=2&delimiter=%2F"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (grouped.statusCode() != 200 || !grouped.body().contains("<CommonPrefixes><Prefix>folder/</Prefix></CommonPrefixes>") ||
+                grouped.body().contains("<Contents>"))
+                throw new AssertionError("Delimiter listing failed: " + grouped.body());
+            var encoded = client.send(signedUri(URI.create(base + "/objects?list-type=2&encoding-type=url"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (encoded.statusCode() != 200 || !encoded.body().contains("folder%2Fmoon-%E2%98%BE.txt"))
+                throw new AssertionError("Encoded listing failed: " + encoded.body());
+            var emptyPage = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=0"),
+                "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            if (emptyPage.statusCode() != 200 || !emptyPage.body().contains("<KeyCount>0</KeyCount>"))
+                throw new AssertionError("Empty list page failed: " + emptyPage.body());
+            String movie = "folder/video.mp4";
+            URI initiate = URI.create(base + "/objects/" + movie + "?uploads=");
+            var created = client.send(signedUri(initiate, "POST", new byte[0],
+                Map.of("content-type", "video/mp4")), HttpResponse.BodyHandlers.ofString());
+            if (created.statusCode() != 200) throw new AssertionError("Multipart initiation failed: " + created.body());
+            String upload = created.body().split("<UploadId>")[1].split("</UploadId>")[0];
+            byte[] first = "hello ".getBytes(StandardCharsets.UTF_8);
+            byte[] second = "world".getBytes(StandardCharsets.UTF_8);
+            var partOne = client.send(signedUri(URI.create(base + "/objects/" + movie +
+                "?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+            var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie +
+                "?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+            status(200, partOne); status(200, partTwo);
+            String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
+                partOne.headers().firstValue("etag").orElseThrow() +
+                "</ETag></Part><Part><PartNumber>2</PartNumber><ETag>" +
+                partTwo.headers().firstValue("etag").orElseThrow() +
+                "</ETag></Part></CompleteMultipartUpload>";
+            status(200, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload),
+                "POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+            var assembled = client.send(signed(base, "GET", movie, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
+            status(200, assembled);
+            if (!"hello world".equals(new String(assembled.body(), StandardCharsets.UTF_8)) ||
+                !"video/mp4".equals(assembled.headers().firstValue("content-type").orElse("")))
+                throw new AssertionError("Completed multipart object mismatch");
+            var abandoned = client.send(signedUri(URI.create(base + "/objects/abandoned?uploads="),
+                "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+            String abandonedId = abandoned.body().split("<UploadId>")[1].split("</UploadId>")[0];
+            status(204, client.send(signedUri(URI.create(base + "/objects/abandoned?uploadId=" + abandonedId),
+                "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
             var head = client.send(signed(base, "HEAD", key, new byte[0]),
                 HttpResponse.BodyHandlers.ofByteArray());
             status(200, head);
@@ -91,7 +174,7 @@ public final class HttpTest {
                 HttpResponse.BodyHandlers.ofByteArray()));
             status(404, client.send(signed(base, "GET", key, new byte[0]),
                 HttpResponse.BodyHandlers.ofByteArray()));
-            System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE");
+            System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE, MIME, ranges, listing, multipart");
         } finally {
             server.stop(0);
             executor.close();

test/cloud/lunarsky/store/StoreTest.java

@@ -2,14 +2,17 @@ package cloud.lunarsky.store;
 import java.nio.file.*;
 import java.io.*;
 import java.util.Arrays;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.util.List;
 
 public final class StoreTest {
     interface Operation {void run() throws Exception;}
     static void fails(int status,Operation operation)throws Exception{
         try{operation.run();throw new AssertionError("Expected "+status);}catch(StoreException error){if(error.status!=status)throw error;}
     }
-    static DiskStore.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
-        return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only);
+    static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
+        return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream");
     }
     public static void main(String[] args)throws Exception{
         var headers=new com.sun.net.httpserver.Headers();
@@ -35,11 +38,13 @@ public final class StoreTest {
                 fails(412,()->put(store,"../nested/☾",new byte[]{9},true));
                 fails(507,()->put(store,"second",body,true));
                 fails(413,()->put(store,"large",new byte[9],true));
-                fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true));
-                fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true));
+                fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true,"application/octet-stream"));
+                fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true,"application/octet-stream"));
                 fails(404,()->store.open("test","bad"));
                 put(store,"../nested/☾",new byte[]{9},false);
                 put(store,"empty",new byte[0],true);
+                if(store.list("test","","",100,null).objects().size()!=2)throw new AssertionError("List index");
+                if(store.objectCount()!=2||store.legacyObjects()!=0)throw new AssertionError("Object counts");
                 try {
                     new DiskStore(root,8,10);
                     throw new AssertionError("A second process opened the same data directory");
@@ -49,9 +54,53 @@ public final class StoreTest {
             }
             try(var restarted=new DiskStore(root,8,10)){
                 try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
+                if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
                 restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
                 fails(404,()->restarted.open("test","../nested/☾"));
+                var uploads=new MultipartStore(restarted);
+                String upload=uploads.create("test","from-parts","text/plain");
+                byte[] part={1,2,3};
+                uploads.putPart(upload,"test","from-parts",1,new ByteArrayInputStream(part),part.length,
+                    SigV4.hex(SigV4.hash(part)),null);
+                Files.writeString(root.resolve("pending-upload-id"),upload);
             }
+            try(var resumed=new DiskStore(root,8,10)){
+                Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000");
+                Files.createDirectory(unfinished);
+                Files.write(unfinished.resolve("manifest"),new byte[]{1,2,3});
+                var uploads=new MultipartStore(resumed);
+                if(Files.exists(unfinished))throw new AssertionError("Unfinished multipart creation survived restart");
+                String upload=Files.readString(root.resolve("pending-upload-id"));
+                byte[] part={1,2,3};
+                String etag=SigV4.hex(MessageDigest.getInstance("MD5").digest(part));
+                uploads.complete(upload,"test","from-parts",List.of(new MultipartStorage.Part(1,etag)));
+                try(var obj=resumed.open("test","from-parts")){
+                    if(!Arrays.equals(part,obj.stream().readAllBytes())||!obj.metadata().contentType().equals("text/plain"))
+                        throw new AssertionError("Multipart restart");
+                }
+                resumed.delete("test","from-parts");
+            }
+            byte[] old={4,5,6};
+            String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
+            Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId);
+            Files.createDirectories(oldPath.getParent());
+            ByteBuffer oldRecord=ByteBuffer.allocate(72+old.length).putLong(0x4c534f424a303031L)
+                .putLong(old.length).putLong(123456789L)
+                .put(MessageDigest.getInstance("MD5").digest(old)).put(SigV4.hash(old)).put(old);
+            Files.write(oldPath,oldRecord.array());
+            try(var migrated=new DiskStore(root,8,10)){
+                if(migrated.objectCount()!=2||migrated.legacyObjects()!=1)throw new AssertionError("Legacy counts");
+                try(var obj=migrated.open("test","legacy")){
+                    if(!Arrays.equals(old,obj.stream().readAllBytes()))throw new AssertionError("Legacy read");
+                }
+                if(migrated.list("test","","",100,null).objects().stream().anyMatch(entry->entry.key().equals("legacy")))
+                    throw new AssertionError("Legacy object appeared without a stored key");
+                put(migrated,"legacy",old,false);
+                if(migrated.objectCount()!=2||migrated.legacyObjects()!=0)throw new AssertionError("Legacy count after overwrite");
+                if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy")))
+                    throw new AssertionError("Legacy overwrite was not indexed");
+            }
+            Files.delete(root.resolve("pending-upload-id"));
             String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
             Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
             try {
@@ -61,7 +110,7 @@ public final class StoreTest {
                 if(!expected.getMessage().contains("object record"))throw expected;
             }
             try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
-            System.out.println("Java storage tests passed: roundtrip, quota, integrity, persistence, locking, corruption, delete");
+            System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete");
         }finally{try(var paths=Files.walk(root)){for(var p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}}
     }
 }