Configure gateway limits and encrypted storage

To-do Update Main.java to reduce Complexity and readability.

.env.cluster.example

@@ -1,7 +1,15 @@
-S3_ACCESS_KEY=
+S3_ACCESS_KEY=[Replace_with_your_S3_Access_Key]
-S3_SECRET_KEY=
+S3_SECRET_KEY=[Replace_with_your_S3_Secret_Key]
 CLUSTER_TOKEN=
 CLUSTER_REPAIR_TOKEN=
-POSTGRES_PASSWORD=
+POSTGRES_PASSWORD=[PG_Pass]
 S3_BUCKET=objects
 CLUSTER_HOST_PORT=9001
+MAX_OBJECT_BYTES=134217728
+MAX_TOTAL_BYTES=2147483648
+MAX_IN_FLIGHT_REQUESTS=16
+HTTP_BACKLOG=64
+S3_VIRTUAL_HOST_SUFFIX=
+# For compose.cluster.encrypted.yaml only; provision and mount LUKS before setting these.
+ENCRYPTED_STORAGE_ROOT=
+ENCRYPTED_VOLUME_ID=

.env.example

@@ -7,3 +7,15 @@ S3_REGION=us-east-1
 HOST_PORT=9000
 MAX_OBJECT_BYTES=134217728
 MAX_TOTAL_BYTES=2147483648
+PUBLIC_REQUESTS_PER_SECOND=0
+PUBLIC_REQUEST_BURST=
+PUBLIC_BYTES_PER_SECOND=0
+PUBLIC_BYTE_BURST=
+PUBLIC_MAX_IN_FLIGHT_PER_IP=
+PUBLIC_TRUSTED_PROXY_IPS=
+MAX_IN_FLIGHT_REQUESTS=16
+HTTP_BACKLOG=64
+S3_VIRTUAL_HOST_SUFFIX=
+# For compose.encrypted.yaml only; provision and mount LUKS before setting these.
+ENCRYPTED_STORAGE_ROOT=
+ENCRYPTED_VOLUME_ID=

Dockerfile

@@ -4,19 +4,26 @@ RUN wget -q -O /tmp/postgresql.jar https://jdbc.postgresql.org/download/postgres
     echo '73914527305a40cce504b0d3d90b23caf565136912d607ae8ae7c5895512332c  /tmp/postgresql.jar' | sha256sum -c -
 COPY src ./src
 COPY test ./test
-RUN mkdir /out && javac --release 21 --add-modules jdk.httpserver,java.net.http -d /out src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java
+COPY lib/hash4j-0.30.0.jar /tmp/hash4j.jar
-RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.StoreTest
+RUN echo 'd3224b113ea835ce3452097747e5209ec760cf443dbed770d6ba68a0d480178f  /tmp/hash4j.jar' | sha256sum -c -
-RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.ConcurrencyTest
+RUN mkdir /out && javac --release 21 --add-modules jdk.httpserver,java.net.http -cp /tmp/hash4j.jar -d /out src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java
-RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.HttpTest
+RUN java --add-modules jdk.httpserver -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.StoreTest
-RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.ClusterNodeTest
+RUN java --add-modules jdk.httpserver -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ConcurrencyTest
-RUN java -cp /out cloud.lunarsky.store.CliTest
+RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.HttpTest
+RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClientLimitsTest
+RUN java -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.EncryptedVolumeTest
+RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClusterNodeTest
+RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClusterTlsTest
+RUN java -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.CliTest
 
 FROM eclipse-temurin:21-jre-alpine
 RUN addgroup -g 10001 store && adduser -D -u 10001 -G store store && mkdir /data && chown store:store /data
 COPY --from=build /out /app
 COPY --from=build /tmp/postgresql.jar /app/postgresql.jar
+COPY --from=build /tmp/hash4j.jar /app/hash4j.jar
+COPY lib/LICENSE.hash4j /app/LICENSE.hash4j
 COPY scripts/objectstore /usr/local/bin/objectstore
 RUN chmod 755 /usr/local/bin/objectstore
 USER store
 EXPOSE 9000
-ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.Main"]
+ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar:/app/hash4j.jar", "cloud.lunarsky.store.Main"]

README.md

@@ -8,7 +8,7 @@ Source: [GitHub](https://github.com/LunarSkyOSS/ObjectStore) · [Gitea mirror](h
 
 **Development software. Do not use it for production data.** It is provided as is, without warranty, under the [MIT License](LICENSE).
 
-[![Stage: development](assets/badge-stage.svg)](#limits-and-safety) [![License: MIT](assets/badge-license.svg)](LICENSE) [![Runtime: Java 21](assets/badge-java.svg)](Dockerfile) [![S3 API: partial support](assets/badge-api.svg)](#s3-api-support-checklist)
+[![Stage: development](assets/badge-stage.svg)](#limits-and-safety) [![License: MIT](assets/badge-license.svg)](LICENSE) [![Runtime: Java 21](assets/badge-java.svg)](Dockerfile) [![S3 API: partial support](assets/badge-api.svg)](#s3-api-support-checklist) [![CodeFactor](https://www.codefactor.io/repository/github/lunarskyoss/objectstore/badge)](https://www.codefactor.io/repository/github/lunarskyoss/objectstore)
 
 ## Contents
 
@@ -16,41 +16,62 @@ Source: [GitHub](https://github.com/LunarSkyOSS/ObjectStore) · [Gitea mirror](h
 - [S3 API support checklist](#s3-api-support-checklist)
 - [Tests](TESTS.md)
 - [Single-node setup](#single-node-setup)
+- [Encrypted storage](#encrypted-storage)
+- [Access keys and ACLs](#access-keys-and-acls)
 - [CLI and tests](#cli-and-tests)
+- [Capability discovery](#capability-discovery)
+- [Java client](#java-client)
 - [Local cluster prototype](#local-cluster-prototype)
+- [Node transport TLS](#node-transport-tls)
+- [Metadata primary routing](#metadata-primary-routing)
 - [Migrating a local cluster](#migrating-a-local-cluster)
 - [Adding a cluster node](#adding-a-cluster-node)
+- [Cluster maintenance and recovery](#cluster-maintenance-and-recovery)
 - [Limits and safety](#limits-and-safety)
 - [Disclaimer](#disclaimer)
 - [AI contributions](#ai-contributions)
 
 ## Capability checklist
 
-ObjectStore serves one configured bucket.
+ObjectStore creates the configured default bucket at startup. Additional buckets share the configured capacity limit.
 
 - ✅ Persistent single-node storage with checksum verification
+- ✅ Optional dm-crypt-backed data mounts with startup guards
 - ✅ Configurable per-object and total logical size limits
 - ✅ CLI status, version, and full payload verification
 - ✅ Local cluster prototype with stable node IDs and host-aware placement code
-- ⬜ Automatic repair, rebalance, and garbage collection
+- ✅ Optional HTTPS between cluster processes and storage nodes
+- ✅ Opt-in automatic repair, rebalance, and guarded garbage collection in the local cluster
+- ✅ Metadata backup and tested restore to a separate local PostgreSQL instance
+- ✅ Manual [two-machine durability and metadata-restore drill](tests/two-host/README.md)
 - ⬜ Production multi-server deployment and metadata failover
 
 New objects retain their content type and key. Objects written by the earlier single-node format remain readable, but cannot appear in listings until overwritten because their original keys were not stored.
 
 ## S3 API support checklist
 
-- ✅ Header-based AWS Signature Version 4 authentication
+- ✅ Header-based and presigned-query AWS Signature Version 4 authentication
+- ✅ Path-style and configurable virtual-hosted-style bucket addresses
 - ✅ `PutObject`, `GetObject`, `HeadObject`, and `DeleteObject` in both modes
 - ✅ Single-range GET and `ListObjectsV2` in both modes
 - ✅ SHA-256 payload verification and `x-amz-checksum-sha256` in both modes
-- ✅ `CreateMultipartUpload`, `UploadPart`, `CompleteMultipartUpload`, and `AbortMultipartUpload` in single-node mode
+- ✅ `Content-MD5` and CRC32, CRC32C, CRC64NVME, XXHash64, XXHash3, XXHash128, SHA-1, SHA-256, SHA-512, and MD5 checksum headers on `PutObject` and `UploadPart`
-- ⬜ Multipart uploads in cluster mode
+- ✅ `CreateMultipartUpload`, `UploadPart`, `CompleteMultipartUpload`, and `AbortMultipartUpload` in both modes
-- ⬜ Presigned URLs and streaming Signature V4 uploads
+- ✅ `ListParts` and `ListMultipartUploads` in both modes
-- ⬜ `CopyObject`, `ListParts`, and `ListMultipartUploads`
+- ✅ Presigned URLs and signed streaming Signature V4 uploads, including signed checksum trailers
-- ⬜ `Content-MD5` and checksum algorithms other than SHA-256
+- ✅ `CreateBucket`, `HeadBucket`, `DeleteBucket`, and `ListBuckets` in both modes
-- ⬜ Bucket creation and listing, object versioning, ACLs, tags, and user metadata
+- ✅ `CopyObject` across owned buckets, with `COPY` and `REPLACE` content-type and user-metadata behavior
+- ✅ User metadata on object and multipart uploads; object tags on upload, copy, and the tagging subresource
+- ✅ CRC64NVME and XXHash checksums, checksum trailers, and persisted object checksum metadata
+- ✅ Bucket versioning with retained versions, delete markers, and version-specific reads, copies, deletes, and tags in both modes
+- ✅ Basic bucket and object ACL grants, public reads, and multiple access-key identities
+- ⬜ Full AWS ACL ownership controls, email grantees, and bucket policies
 
 This is an S3 API subset, not full AWS S3 compatibility. Unsupported S3 operations and Amazon-specific headers are rejected.
+Buckets use the same three-to-63-character lowercase names as the configured default bucket. The default bucket cannot be deleted through the API. Bucket creation currently accepts the empty-body request used for the configured region. Tags do not control access.
+Versioning supports enabled and suspended states, historical object versions, null versions, and delete markers. Retained versions count toward the capacity limit. Deleting a specific version is permanent. Lifecycle expiration, MFA Delete, and `ListObjectVersions` delimiter grouping are not supported yet.
+User metadata values currently accept printable ASCII only; non-ASCII metadata header encoding is not yet supported.
+Checksums supplied with `PutObject` are validated before publication and retained across restarts, copies, and object versions. When no checksum is supplied, ObjectStore calculates and stores CRC64NVME, including for completed multipart uploads. Request `x-amz-checksum-mode: ENABLED` on `GetObject` or `HeadObject` to receive the stored checksum. `UploadPart` checksums are validated but are not yet returned by `ListParts` or combined into a multipart checksum; the completed object's CRC64NVME covers its full content. Presigned URLs use query Signature V4 with a maximum seven-day expiry. Streaming uploads support signed `aws-chunked` payloads and one signed checksum trailer. Temporary credentials and other streaming payload modes remain unsupported. Copies use the existing object size limit.
 
 ## Single-node setup
 
@@ -66,6 +87,31 @@ Port 9000 binds to localhost. Data stays in the `object-data` Docker volume. `do
 
 The standalone defaults are 128 MiB per object and 2 GiB total. Set `MAX_OBJECT_BYTES` and `MAX_TOTAL_BYTES` in `.env` to change them. Incomplete multipart uploads consume space until aborted.
 
+## Encrypted storage
+
+At-rest encryption is an **opt-in deployment setting** backed by a host-mounted dm-crypt/LUKS filesystem. ObjectStore does not encrypt bytes itself or store an encryption key in its environment. Prepare and unlock the LUKS filesystem outside Docker, then set `ENCRYPTED_STORAGE_ROOT` to its exact mount point and `ENCRYPTED_VOLUME_ID` to a stable 32-character lowercase hex identifier in your private environment file. Generate the identifier with `openssl rand -hex 16`; it is a mount guard, **not** a cryptographic key. Keep the LUKS recovery key separately from the data and backups.
+
+With the encrypted filesystem mounted, prepare empty directories using `sh scripts/prepare-encrypted-storage.sh single /path/to/objectstore.env` or `sh scripts/prepare-encrypted-storage.sh cluster /path/to/cluster.env`. The script reads only the two encryption settings from that file; it does not execute it. It requires an active dm-crypt-backed mount at `ENCRYPTED_STORAGE_ROOT` and refuses to mark a nonempty directory. Run it with permission to create the directories and set container ownership. Enable the matching Compose override:
+
+```sh
+docker compose --env-file /path/to/objectstore.env -f compose.yaml -f compose.encrypted.yaml up -d --build
+docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml -f compose.cluster.encrypted.yaml up -d --build
+```
+
+Use the first command for single-node mode or the second for the **local development cluster**, not both. The overrides bind encrypted directories for object data, cluster nodes, PostgreSQL, and cluster staging files. Each process checks a marker stored on its mounted directory before opening data; PostgreSQL checks before starting. If a mount is missing after reboot, the service refuses to start instead of silently creating a new plaintext store. Provision an unlock-and-mount service before starting Compose; a container restart cannot unlock LUKS. The preparation script checks the host mount, while the runtime marker is only a guard against an absent or wrong mount.
+
+**Existing Docker volumes are not migrated automatically.** Stop the stack, take and verify a backup, prepare the empty encrypted directories, then copy the stopped volumes into their corresponding directories before starting with the override. For PostgreSQL, copy the existing data directory into `cluster/metadata/pgdata` because the encrypted override changes `PGDATA`. Verify the restored objects and database before retiring the original volumes. The cluster overlay places all local test containers on one encrypted host; a future multi-host deployment needs an encrypted data mount and key recovery plan on each host.
+
+This covers the specified data and staging mounts, but not Docker logs, host swap, other temporary files, external backups, or a compromised running host. Encrypt those separately as appropriate, use TLS for network paths, restrict access, and test backup recovery. Encryption at rest is one security measure; enabling it does not by itself establish GDPR compliance.
+
+## Access keys and ACLs
+
+`S3_ACCESS_KEY` is the owner identity. Its secret is `S3_SECRET_KEY`. Additional keys are optional: place one `ACCESSKEY:secret` pair per line in a file mounted read-only inside the container, and set `S3_CREDENTIALS_FILE=/run/secrets/s3-credentials` in the environment file. Use a Compose override to mount an absolute host path at `/run/secrets/s3-credentials` for the `objectstore` service (or `gateway` in cluster mode). Each access key needs 16–128 alphanumeric characters and each secret at least 32 characters. A restart loads changes to that file. Keep it outside Git and protect it as a secret. Additional identities have no access until the owner grants it.
+
+The owner can send `x-amz-acl: public-read` or signed `x-amz-grant-*` headers on bucket creation, object uploads, copies, and multipart initiation. `GET` and `PUT ?acl` support bucket and object ACLs; a PUT accepts either signed grant headers with an empty body or an XML `AccessControlPolicy`. A grantee ID is its configured access key. Supported permissions are `READ`, `WRITE`, `READ_ACP`, `WRITE_ACP`, and `FULL_CONTROL`. The `AllUsers` group is limited to `READ`; `AuthenticatedUsers` is also recognized. Anonymous reads work only where `AllUsers` has a read grant. Replacing an object starts with a private ACL unless the new upload supplies grants; older version ACLs remain attached to their versions.
+
+This is a deliberately limited ACL subset. The configured owner owns every bucket and object. A `WRITE_ACP` grantee can change an object ACL only on a retained, non-null version; updates to current unversioned and null versions require the owner key. List-versions, multipart inspection, tagging, versioning controls, bucket creation/deletion, and the capability endpoint remain owner-only. There are no IAM policies, email grantees, Object Ownership modes, Block Public Access settings, or temporary credentials. Granting public bucket `READ` exposes object names through `ListObjectsV2`; granting public object `READ` exposes that object's bytes. Review those grants before exposing a gateway to the internet.
+
 ## CLI and tests
 
 From the server shell, run the CLI inside the running container from the directory containing `compose.yaml`:
@@ -78,11 +124,23 @@ docker compose exec objectstore objectstore version
 
 The startup log shows the LunarSky banner, version, and a small storage summary (`docker compose logs --tail=20 objectstore`). `status` reports object and multipart usage. `verify` also checks stored payload hashes and exits nonzero on an error. Both commands can run while the service is live; they are not a snapshot or a backup.
 
+## Capability discovery
+
+ObjectStore provides a signed `GET /_objectstore/capabilities` endpoint. Use the same header-based Signature V4 authentication as the S3 API. It returns JSON with `schemaVersion: 1`, the service and software version, storage mode, supported operation names, and configured limits. The endpoint does not disclose credentials or cluster topology. The response has `Cache-Control: no-store` because limits can change after a restart.
+
+`operations` lists implemented API operations, not every AWS option for each operation or the caller's authorization to use them. `limits.maxObjectBytes` applies to a completed object and to each uploaded part; `limits.maxTotalBytes` is the logical storage limit; `limits.maxParts` is 10,000. Future schema version 1 responses may add fields. Clients should ignore unknown fields and treat unknown operation names as unsupported by their own implementation. This endpoint is an ObjectStore extension; a missing endpoint on another S3-compatible service does not prove that a feature is unavailable.
+
 Run `sh scripts/test.sh` with JDK 21 to test from source. See [TESTS.md](TESTS.md) for coverage, the disposable Docker cluster suite, and the limits of those tests.
 
+The server includes [hash4j](https://github.com/dynatrace-oss/hash4j) for streaming XXHash checksums. Its Apache-2.0 license is included at [lib/LICENSE.hash4j](lib/LICENSE.hash4j).
+
+## Java client
+
+The [JDK-only Java client](client/README.md) works with ObjectStore and other S3-compatible endpoints. It supports object transfers, listing, multipart uploads, and read-only capability discovery. An [AWT image manager](client/examples/README.md) provides a small desktop example. Run `bash client/scripts/build.sh` to produce its JAR and Javadoc locally.
+
 ## Local cluster prototype
 
-The local cluster prototype starts three segment containers and one PostgreSQL container on the same Docker host. Copy `.env.cluster.example` to a private environment file, replace all four credentials, and run:
+The local cluster prototype starts three segment containers and one PostgreSQL container on the same Docker host. Copy `.env.cluster.example` to a private environment file, replace all five credential values, and run:
 
 ```sh
 docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml up -d --build
@@ -92,8 +150,37 @@ docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml run --rm
 
 The cluster S3 endpoint binds to `127.0.0.1:9001`; storage nodes and PostgreSQL have no published ports. The separate repair container holds the repair credential and restores missing or corrupt replicas.
 
+Multipart parts are stored on cluster nodes and indexed in PostgreSQL. Incomplete uploads count toward the logical capacity limit; abort them to release that capacity. Repair includes staged parts. The gateway upgrades the metadata schema when it starts, so back up the database before upgrading an existing cluster.
+
 Node UUIDs persist on their volumes, and replica manifests use those UUIDs so reordering configured URLs cannot move an existing replica. Each node also has an operator-assigned physical host UUID. New writes require acknowledgements from two different host UUIDs. The optional `CLUSTER_TEST_NODE_DOMAINS=true` override counts containers instead, solely for local process tests; all containers in this Compose file share one physical host.
 
+## Node transport TLS
+
+The default local Compose cluster uses HTTP inside its private Docker network. For an HTTPS test, give each node a PKCS#12 keystore containing its private key and a certificate whose DNS subject alternative name matches its `CLUSTER_NODES` hostname. Give the gateway, repair, garbage collection, and maintenance processes a PKCS#12 truststore containing the issuing CA or each node certificate. Mount the files read-only and keep the keystores and password files outside Git.
+
+Set `NODE_TLS_KEYSTORE` and `NODE_TLS_PASSWORD_FILE` on each node. Set `CLUSTER_TLS_TRUSTSTORE` and `CLUSTER_TLS_PASSWORD_FILE` on every process that contacts nodes, and change each node URL to `https://`. With a truststore configured, HTTP node URLs are rejected. The client verifies the certificate chain and hostname; a failed handshake does not fall back to HTTP. Both settings in each pair are required. Restart affected processes after rotating certificates or truststores.
+
+The optional [Compose TLS overlay](compose.cluster.tls.yaml) expects `node-a.p12` through `node-d.p12`, matching `.pass` files, and `trust.p12` with `trust.pass` in `CLUSTER_TLS_DIR`. Set that variable to a private certificate directory and include both Compose files:
+
+```sh
+CLUSTER_TLS_DIR=/private/objectstore-certs docker compose --env-file /path/to/cluster.env \
+  -f compose.cluster.yaml -f compose.cluster.tls.yaml up -d --build
+```
+
+The files must be readable by container UID 10001 without making private keys or passwords world-readable. Add HTTPS URLs for additional nodes when expanding the cluster.
+
+This secures node traffic only. The local cluster still lacks automatic PostgreSQL failover, database TLS configuration, encryption at rest, and production multi-server validation. Its HTTP S3 gateway remains bound to localhost; use a separate trusted proxy for external TLS. Do not treat the TLS overlay as a production deployment.
+
+## Metadata primary routing
+
+`POSTGRES_JDBC_URL` can override the database URL for the gateway, repair, garbage collection, and maintenance processes. Its local Compose default now uses `targetServerType=primary`, and `/ready` returns unavailable when the database is read-only or in recovery. The base Compose file still starts and waits for its own single PostgreSQL container; it is not an HA deployment. In an independently managed deployment, list the PostgreSQL hosts in the JDBC URL and keep `targetServerType=primary`:
+
+```text
+jdbc:postgresql://db-a:5432,db-b:5432/objectstore?targetServerType=primary&connectTimeout=3&socketTimeout=10
+```
+
+ObjectStore opens a new database connection for each operation, so the JDBC driver can select a promoted primary after the old one is stopped. This does **not** promote a standby, fence the old primary, configure synchronous replication, or guarantee that a recently acknowledged write reached the standby. Those jobs belong to a separately operated PostgreSQL HA system. Never allow two writable metadata databases: they can diverge while serving different ObjectStore requests. A request interrupted during failover has an uncertain outcome; verify it before retrying a non-idempotent operation. For remote database connections, configure PostgreSQL TLS and use JDBC `sslmode=verify-full` with a mounted CA certificate. The provided local Compose database does not enable TLS.
+
 ## Migrating a local cluster
 
 For an existing **local** three-node cluster that stores replicas by URL position:
@@ -108,11 +195,35 @@ The old format did not record the original URL mapping, so the inventory check i
 
 ## Adding a cluster node
 
-`objectstore cluster-join http://new-node:9100 expected-host-uuid` registers an additional local node. Add its URL to `CLUSTER_NODES` and restart the gateway to use it for new writes. Existing segments stay where their manifests say; this is capacity expansion for new writes, not a rebalance. `scripts/test-cluster.sh` exercises a fourth container joining and receiving new segments.
+`objectstore cluster-join http://new-node:9100 expected-host-uuid` registers an additional local node. Add its URL to `CLUSTER_NODES` and restart the gateway. A repair pass then copies existing segments to their preferred nodes and removes obsolete replicas from the manifest only after verifying the replacements. `scripts/test-cluster.sh` exercises a fourth container joining, receiving new segments, and rebalancing existing ones.
+
+## Cluster maintenance and recovery
+
+From the Compose directory, use `docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml` as the command prefix:
+
+```sh
+docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml --profile automatic up -d maintenance
+docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml run --rm gc
+sh scripts/backup-cluster-metadata.sh /path/to/cluster.env /path/to/metadata.dump
+```
+
+The maintenance service is opt-in. It repairs missing or corrupt replicas and rebalances them every 60 seconds by default. Set `CLUSTER_MAINTENANCE_INTERVAL_SECONDS` to change the interval. The `gc` command is a dry run; use `gc --apply` only after checking its candidate count and keeping independent backups. Cleanup records each orphan on one pass and waits at least `CLUSTER_GC_MIN_AGE_SECONDS` before deleting it on a later pass. The default age is 14 days. To permit deletion, set `CLUSTER_BACKUP_RETENTION_SECONDS` to your actual backup retention in seconds; it must be at least one day and shorter than the cleanup age. Scheduled cleanup also requires `CLUSTER_GC_ENABLED=true` on the maintenance service. The backup command writes a verified PostgreSQL archive with private file permissions. Restore it to a separate database and point a gateway at that database only after validating the restore. A backup restore is manual recovery, not automatic failover.
 
 ## Limits and safety
 
-The cluster retains old and failed-write segments. It has no garbage collection, metadata standby, automated rebalance, private-network TLS, scoped credentials, or physical host verification yet. Host UUIDs are operator labels, not proof that machines have separate power, disks, or network paths. Keep `CLUSTER_LOCAL_DEV=true` limited to local tests.
+Public-client limits are **disabled by default**. The localhost Compose setup is unchanged. For an endpoint that accepts external clients, set one or both of `PUBLIC_REQUESTS_PER_SECOND` and `PUBLIC_BYTES_PER_SECOND` to a positive number in `.env`. The first limits requests per client IP with a token bucket; the second paces upload and download bytes through one shared per-IP budget. `PUBLIC_REQUEST_BURST` and `PUBLIC_BYTE_BURST` default to one second of their respective rates. `PUBLIC_MAX_IN_FLIGHT_PER_IP` defaults to 8 when either rate is enabled. Requests above the rate or concurrency limit receive S3 `503 SlowDown` and `Retry-After: 1`; an admitted transfer is paced rather than cut off. These are per-gateway limits, not cluster-wide quotas.
+
+`MAX_IN_FLIGHT_REQUESTS` controls the per-gateway concurrency cap (default 16, range 1–1024); an additional request receives `503 SlowDown`. `HTTP_BACKLOG` controls the listening socket backlog (default 64, range 1–4096). Raise either only after a mixed upload/download load test, because each active request can hold memory, disk space, and database connections. The configured object and total storage limits still apply. `MAX_OBJECT_BYTES` currently cannot exceed 1 GiB.
+
+Set `S3_VIRTUAL_HOST_SUFFIX` to a DNS suffix such as `s3.example.com` to accept `bucket.s3.example.com/key` alongside path-style `/bucket/key`. Configure DNS and TLS for the bucket hostnames, and preserve the client's original `Host` header through the proxy; Signature V4 signs it. The suffix is empty by default. This changes request parsing, not bucket naming rules or DNS configuration.
+
+For example, to start with 100 requests per second and 16 MiB/s combined upload and download per IP, set `PUBLIC_REQUESTS_PER_SECOND=100` and `PUBLIC_BYTES_PER_SECOND=16777216`. Both start with a one-second burst. Adjust these numbers after measuring the actual workload; do not copy them as a universal production policy.
+
+ObjectStore uses the socket peer as the client IP and ignores forwarded-IP headers by default. If a reverse proxy sits between external clients and ObjectStore, set `PUBLIC_TRUSTED_PROXY_IPS` to the exact IP address that ObjectStore sees for that proxy and configure the proxy to **replace** `X-Real-IP` with its actual client IP. A request from that trusted peer without exactly one valid numeric `X-Real-IP` is rejected. Do not trust an address reachable by arbitrary clients, and preserve the original `Host` header for Signature V4. In Docker, the proxy's address as seen by the container may be a bridge address rather than `127.0.0.1`. If proxy trust is not configured, all clients behind that proxy share its budget; this is safe from header spoofing but may throttle them together.
+
+Enable these limits only on a deliberately public endpoint. They also apply to direct localhost storage calls to that same endpoint; leave them disabled for the local-only setup or run a separate local-only instance if local storage calls must be exempt. A direct loopback `/health` probe without a forwarded-IP header remains exempt. The byte limit is aggregate ingress plus egress for each IP, and several users behind one NAT share it. If a proxy buffers complete uploads before forwarding them, the upload byte limit controls proxy-to-ObjectStore traffic, not the client's initial upload speed; disable request buffering when end-to-end upload pacing is required. It is a fairness control, not a defense against connection floods before the Java handler runs. Put an internet-facing proxy or firewall in front of the gateway for TLS, connection limits, request timeouts, and buffering controls. Do not expose storage nodes or PostgreSQL publicly.
+
+The default local cluster still uses plaintext node and PostgreSQL connections. The optional TLS overlay secures node traffic, but PostgreSQL TLS, automatic metadata failover, scoped credentials, and physical host verification remain absent. Garbage collection can remove data required by an older metadata backup, so its retention guard is essential. Host UUIDs are operator labels, not proof that machines have separate power, disks, or network paths. Keep `CLUSTER_LOCAL_DEV=true` limited to local tests.
 
 The standalone cluster node binds to localhost by default. Set `NODE_BIND` only for a private test network; the Compose file binds inside its private Docker network. PostgreSQL JDBC 42.7.14 is bundled in the image with its license inside the JAR.
 

TESTS.md

@@ -16,17 +16,27 @@ The script compiles the source and test programs into `out/classes`, then runs:
 
 | Test | Checks |
 | --- | --- |
-| `StoreTest` | Signature V4 test vector and tampering, local writes and reads, quotas, restart persistence, multipart recovery, legacy reads, locking, and corruption rejection. |
+| `StoreTest` | Signature V4 and signed-stream vectors, CRC64NVME and XXHash reference vectors, local writes and reads, quotas, restart persistence for objects, checksums, ACLs, attributes, buckets, and versions, multipart recovery, legacy reads, locking, and corruption rejection. |
 | `ConcurrencyTest` | Atomic local overwrites and consistent reads, listings, and deletes during concurrent access. |
-| `HttpTest` | Signed HTTP requests, object operations, ranges, listing, and single-node multipart uploads. |
+| `HttpTest` | Signed capability discovery, presigned URLs, streaming uploads and trailers, object and bucket operations, ACL grants with a second access key and public reads, copies, checksum persistence and rejection, ranges, listing, metadata, tags, multipart uploads, versioning, and signed virtual-hosted requests in single-node mode. |
-| `ClusterNodeTest` | Node identity and locking, authenticated segment transfers, checksum rejection, repair authorization, and restart cleanup. |
+| `ClientLimitsTest` | Disabled defaults, trusted-proxy address validation, ignored untrusted headers, per-IP request refusal, and paced response bytes. |
+| `EncryptedVolumeTest` | Marker acceptance and refusal when the configured encrypted directory is missing, mismatched, or a symlink. |
+| `ClusterNodeTest` | Node identity and locking, authenticated segment transfers, checksum rejection, repair authorization, inventory and guarded deletion, and restart cleanup. |
+| `ClusterTlsTest` | HTTPS node identity and segment roundtrip with a trusted certificate, plus rejection of untrusted and wrong-host certificates. |
 | `CliTest` | Version, status, verification, and a nonzero result for corrupt data. |
+| `ClientTest` and `MultipartClientTest` | Java client request signing, capability discovery, error handling, and multipart operations. |
 
-The script exits nonzero on failure. The test programs use temporary local directories and loopback HTTP ports; they do not use an existing ObjectStore volume.
+The script exits nonzero on failure. The test programs use temporary local directories and loopback HTTP or HTTPS ports; they do not use an existing ObjectStore volume. `ClusterTlsTest` uses the JDK's `keytool` to create disposable test certificates.
+
+The encrypted Compose overlays can be checked with `docker compose config` without starting the stack. This verifies their bind mounts and startup settings, not that a host filesystem is actually encrypted. `scripts/prepare-encrypted-storage.sh` must also verify an active dm-crypt-backed mount. A complete deployment drill must unlock it, prepare directories, start with the overlay, write data, restart, restore from backup, and confirm that the stack refuses to start while the mount is unavailable. Do not point that drill at existing data.
+
+## Disposable metadata routing test
+
+Run `sh scripts/test-metadata.sh` with Docker Compose. It creates a separate, temporary PostgreSQL container and two in-process storage nodes. The test connects through a two-host JDBC URL whose first host is unavailable, checks writable readiness, then makes the test database read-only and verifies that readiness drops. The script removes its test containers and temporary database afterward. It does not promote a standby or test automatic failover.
 
 ## Disposable Docker cluster tests
 
-Requires Docker with Compose, Python 3, `curl`, and a free local port 9001. Make a test-only environment file from `.env.cluster.example` and fill in all five blank credentials with test-only values. Keep that file private and out of Git.
+Requires Docker with Compose, Python 3.9 or newer, `curl`, and a free local port 9001. Make a test-only environment file from `.env.cluster.example` and replace all five credential values with test-only values. Keep that file private and out of Git.
 
 ```sh
 cp .env.cluster.example /tmp/objectstore-cluster-tests.env
@@ -42,19 +52,23 @@ COMPOSE_PROJECT_NAME=objectstore-tests sh scripts/test-cluster.sh /tmp/objectsto
 Use a fresh, disposable Compose project. The script writes test objects, stops and restarts storage nodes and PostgreSQL, corrupts a replica to exercise repair, and joins a fourth node. It leaves the test stack running. To remove **only that test project's** containers and volumes after review:
 
 ```sh
-COMPOSE_PROJECT_NAME=objectstore-tests docker compose --env-file /tmp/objectstore-cluster-tests.env -f compose.cluster.yaml --profile expansion down -v
+COMPOSE_PROJECT_NAME=objectstore-tests docker compose --env-file /tmp/objectstore-cluster-tests.env -f compose.cluster.yaml --profile expansion --profile automatic --profile recovery down -v
 ```
 
 If port 9001 is occupied, set `CLUSTER_HOST_PORT` to the same free port in both the environment file and the shell before running the script. The script reads that port from the shell; Compose reads it from the file.
 
-The Docker suite checks signed S3 operations, multi-segment objects, concurrent overwrites, reads and writes with a node stopped, refusal to write without a storage quorum, restart recovery, corrupt-replica repair, metadata unavailability, and placement on a newly joined node. It also checks that containers labeled as one physical host cannot satisfy the normal host quorum. Its local-only override permits the remaining phases to use containers as separate test domains.
+The Docker suite checks signed capability discovery and S3 operations, bucket creation and deletion, metadata and tags, public-read ACLs and anonymous access, copies, upload checksums, multi-segment objects, concurrent overwrites, multipart staging and listings, versioned reads and delete markers, versioned multipart completion, completion after a gateway restart and node loss, reads and writes with a node stopped, refusal to write without a storage quorum, restart recovery, corrupt-replica repair including staged parts, metadata unavailability, read-only metadata readiness rejection, and placement on a newly joined node. It then checks rebalance to the fourth node, automatic repair, garbage collection dry run and delayed deletion, and a metadata backup restored to a separate PostgreSQL instance while the original database is stopped. A two-host JDBC URL selects that restored writable database. Historical regular and multipart versions are checked after repair and cleanup. It also checks that containers labeled as one physical host cannot satisfy the normal host quorum. Its local-only override permits the remaining phases to use containers as separate test domains.
 
 `ClusterMigrationTest` is a separate legacy-format fixture and is **not** run by either test script. Do not run its `create` phase against a populated metadata database. The migration procedure is in the [README](README.md#migrating-a-local-cluster).
 
+## Two-machine durability drill
+
+The [manual two-machine drill](tests/two-host/README.md) uses machine A and machine B with disposable volumes. It records acknowledged writes during a machine interruption, checks reads and write rejection with either storage machine unavailable, restarts test containers during writes, and restores a metadata backup on machine B. This drill is not part of `scripts/test.sh` because it requires two machines and a coordinated interruption.
+
 ## What these tests do not prove
 
 - Container stops are not physical power cuts or disk failures. The automated suite does not reboot a host or test every possible crash point.
 - The Compose nodes share one machine. Passing the local-only quorum override does not demonstrate durability across independent hosts, racks, or sites.
-- The suite does not test metadata failover, an off-site backup restore, prolonged load, or full AWS S3 compatibility.
+- The suite does not test automatic metadata failover, an off-site backup restore, prolonged load, or full AWS S3 compatibility.
 
 See [Limits and safety](README.md#limits-and-safety) before evaluating any multi-server deployment.

client/.gitignore

@@ -0,0 +1 @@
+dist/

client/README.md

@@ -0,0 +1,58 @@
+# ObjectStore Java client
+
+A small, synchronous S3-compatible client built with the JDK alone. It uses path-style URLs and AWS Signature Version 4. The code is covered by the repository's MIT license.
+
+## Build and test
+
+Requires JDK 21 or newer. No Maven, Gradle, or third-party Java libraries are needed.
+
+```sh
+bash client/scripts/test.sh
+bash client/scripts/build.sh
+```
+
+Run these commands from the ObjectStore repository root. The second command writes `client/dist/objectstore-client.jar` and `client/dist/javadoc/`.
+
+The [AWT image manager](examples/README.md) is a small drag-and-drop app for trying uploads, listing, previews, downloads, and deletes against an existing bucket.
+
+## Use
+
+```java
+import cloud.lunarsky.objectstore.client.ObjectStorageClient;
+import cloud.lunarsky.objectstore.client.ObjectStorageClientBuilder;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+
+try (ObjectStorageClient storage = new ObjectStorageClientBuilder()
+        .endpoint(URI.create("https://storage.example.com"))
+        .region("us-east-1")
+        .credentials(accessKey, secretKey)
+        .build()) {
+    storage.putObject("photos", "hello.txt", "hello".getBytes(StandardCharsets.UTF_8), "text/plain");
+    try (ObjectStorageClient.ObjectData data = storage.getObject("photos", "hello.txt")) {
+        data.body().transferTo(System.out);
+    }
+}
+```
+
+`getObject` returns an open stream. Close it even if you do not read every byte. The client requires HTTPS unless you explicitly call `allowInsecureHttp()` for a trusted local endpoint.
+
+## Scope
+
+This version implements single-request object PUT, GET, HEAD, DELETE, ListObjectsV2 pages, standard S3 multipart initiation/part upload/list/complete/abort, unfinished-upload listing, bucket/object ACL GET and PUT, and service detection. ACLs are S3 ACL requests, not a calculation of effective permissions from IAM policies or bucket policies. An S3 service with ACLs disabled can reject them; ObjectStore implements a limited ACL subset.
+
+`getCapabilities()` reads ObjectStore's signed `GET /_objectstore/capabilities` response when available. It reports implemented S3 operation names, service version, storage mode, and configured limits. A listed operation means the service implements it; it does not establish that the current key may perform it or that every AWS option is supported. Older ObjectStore builds fall back to `/health` and leave unverified operations `UNKNOWN`. The response is not cached.
+
+For a generic S3 endpoint, successful operations on this client are recorded as `SUPPORTED`; all untested operations remain `UNKNOWN`. Call `probeReadOnlyCapabilities(bucket, existingObjectKey)` to opt into safe read probes. An S3 denial or missing object leaves that operation `UNKNOWN`, not `UNSUPPORTED`. The client never infers features from a hostname or vendor header. The older `/health` identity fallback is self-reported, not cryptographic attestation.
+
+Multipart upload sessions expose their bucket, key, and upload ID so a caller can persist them and inspect already uploaded parts after a restart. `uploadFileParts` sends a file in sequential parts and leaves completion to the caller. If it fails, the upload remains open for retry or explicit abort; keep the returned upload ID and do not change the source file.
+
+The client does not yet implement presigned URLs, credential providers, automatic retries, region redirects, or streaming uploads of unknown length. A single-request file upload hashes the file before sending it; do not modify the file while the upload runs. The client does not retry writes because a lost response can leave their outcome uncertain.
+
+## Errors
+
+- `ObjectStorageException`: an HTTP error, with status, S3 error code, request ID, and a retryability hint.
+- `TransportException`: connection or I/O failure. A write may already have completed.
+- `ProtocolException`: an unexpected or malformed successful response.
+
+The client does not include access keys, secret keys, or response bodies in exception messages.

client/examples/README.md

@@ -0,0 +1,37 @@
+# Examples
+
+## AWT image manager
+
+![Image manager previewing a cat photo in the local test service](awt-images/screenshot.png)
+
+Cat photo in the screenshot: [IOP Publishing source image](https://ioppublishing.org/wp-content/uploads/2017/03/cat-web-cc0.jpg).
+
+Run from the ObjectStore repository root with JDK 21 or newer:
+
+```sh
+bash client/examples/awt-images/run.sh
+```
+
+The example uses only the Java client and the JDK. Enter an S3-compatible endpoint, region, existing bucket, and access keys. Use **Connect** to verify listing access. Drag PNG, JPEG, GIF, or BMP files onto the window, or use **Add images**. Select an object to preview it; use **Download** or **Delete** to manage it. Objects go under the specified key prefix with a short random ID, so uploading the same filename does not silently replace an earlier image. The list loads 100 objects at a time.
+
+The example does not create a bucket. When launched directly, it keeps credentials in memory and requires explicit opt-in for plain HTTP. Use HTTP only with a trusted test service. Uploads are limited to 64 MiB per file, and previews to 12 MiB. This is a small interoperability test app, not a production asset manager.
+
+### Local Docker test
+
+Prerequisites: JDK 21 or newer, Bash, Python 3, `curl`, and a graphical desktop session. Docker must be running and accessible to your user, and `127.0.0.1:9002` must be free for the test container.
+
+To try the app with a separate ObjectStore service, run:
+
+```sh
+bash client/examples/awt-images/run-test.sh
+```
+
+The launcher builds its local image if needed, starts a test container on `127.0.0.1:9002`, and opens the app connected to a `photos` bucket. Set `OBJECTSTORE_TEST_IMAGE` to use a different image. Closing the app leaves the container and its dedicated data volume running; run the launcher again to reconnect. Its generated test credentials are stored in the container's Docker configuration.
+
+When finished, remove only this example's container and volume:
+
+```sh
+docker stop objectstore-image-example
+docker rm objectstore-image-example
+docker volume rm objectstore-image-example-data
+```

client/examples/awt-images/ImageManager.java

@@ -0,0 +1,508 @@
+import cloud.lunarsky.objectstore.client.Capabilities;
+import cloud.lunarsky.objectstore.client.ObjectStorageClient;
+import cloud.lunarsky.objectstore.client.ObjectStorageClientBuilder;
+
+import java.awt.BorderLayout;
+import java.awt.Button;
+import java.awt.Canvas;
+import java.awt.Checkbox;
+import java.awt.Color;
+import java.awt.Dialog;
+import java.awt.Dimension;
+import java.awt.EventQueue;
+import java.awt.FileDialog;
+import java.awt.FlowLayout;
+import java.awt.Font;
+import java.awt.Frame;
+import java.awt.Graphics;
+import java.awt.GridLayout;
+import java.awt.Label;
+import java.awt.Panel;
+import java.awt.TextField;
+import java.awt.dnd.DnDConstants;
+import java.awt.dnd.DropTarget;
+import java.awt.dnd.DropTargetAdapter;
+import java.awt.dnd.DropTargetDropEvent;
+import java.awt.datatransfer.DataFlavor;
+import java.awt.event.WindowAdapter;
+import java.awt.event.WindowEvent;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
+import javax.imageio.stream.MemoryCacheImageInputStream;
+
+/** A small, dependency-free image browser for testing an S3-compatible endpoint. */
+@SuppressWarnings("serial")
+public final class ImageManager extends Frame {
+    private static final long MAX_UPLOAD = 64L * 1024 * 1024;
+    private static final int MAX_PREVIEW = 12 * 1024 * 1024;
+    private static final Color BACKGROUND = new Color(29, 27, 38);
+    private static final Color FOREGROUND = new Color(231, 223, 240);
+
+    private final TextField endpoint = new TextField(env("S3_ENDPOINT", "http://localhost:9000"));
+    private final TextField region = new TextField(env("S3_REGION", "us-east-1"));
+    private final TextField bucket = new TextField(env("S3_BUCKET", "photos"));
+    private final TextField prefix = new TextField("images/");
+    private final TextField access = new TextField(env("S3_ACCESS_KEY", ""));
+    private final TextField secret = new TextField(env("S3_SECRET_KEY", ""));
+    private final Checkbox allowHttp = new Checkbox("Allow HTTP for trusted tests", null,
+        Boolean.parseBoolean(env("S3_ALLOW_HTTP", "false")));
+    private final java.awt.List images = new java.awt.List(15, false);
+    private final ImageCanvas preview = new ImageCanvas();
+    private final Label status = new Label("Enter a bucket and credentials, then connect. Drop images to upload.");
+    private final Button connectButton = new Button("Connect");
+    private final Button moreButton = new Button("Load more");
+    private final ExecutorService io = Executors.newSingleThreadExecutor(r -> {
+        Thread thread = new Thread(r, "objectstore-image-example");
+        thread.setDaemon(true);
+        return thread;
+    });
+    private final AtomicLong previewRevision = new AtomicLong();
+    private final List<ObjectStorageClient.ObjectEntry> entries = new ArrayList<>();
+    private volatile ObjectStorageClient client;
+    private volatile String activeBucket;
+    private volatile String activePrefix;
+    private String nextToken;
+
+    private ImageManager() {
+        super("ObjectStore image manager");
+        setLayout(new BorderLayout(8, 8));
+        setBackground(BACKGROUND);
+        setForeground(FOREGROUND);
+        setSize(920, 640);
+        setMinimumSize(new Dimension(650, 440));
+        setLocationRelativeTo(null);
+        secret.setEchoChar('\u2022');
+
+        Panel serviceFields = new Panel(new GridLayout(2, 4, 8, 3));
+        serviceFields.add(label("Endpoint"));
+        serviceFields.add(label("Region"));
+        serviceFields.add(label("Bucket"));
+        serviceFields.add(label("Key prefix"));
+        serviceFields.add(endpoint);
+        serviceFields.add(region);
+        serviceFields.add(bucket);
+        serviceFields.add(prefix);
+        Panel credentialFields = new Panel(new GridLayout(2, 2, 8, 3));
+        credentialFields.add(label("Access key"));
+        credentialFields.add(label("Secret key"));
+        credentialFields.add(access);
+        credentialFields.add(secret);
+        Panel connection = new Panel(new BorderLayout(0, 5));
+        connection.add(serviceFields, BorderLayout.NORTH);
+        connection.add(credentialFields, BorderLayout.CENTER);
+
+        Panel actions = new Panel(new FlowLayout(FlowLayout.LEFT, 8, 4));
+        Button upload = new Button("Add images...");
+        Button refresh = new Button("Refresh");
+        Button download = new Button("Download");
+        Button delete = new Button("Delete");
+        actions.add(allowHttp);
+        actions.add(connectButton);
+        actions.add(upload);
+        actions.add(refresh);
+        actions.add(moreButton);
+        actions.add(download);
+        actions.add(delete);
+        moreButton.setEnabled(false);
+
+        Panel top = new Panel(new BorderLayout(0, 5));
+        top.add(connection, BorderLayout.CENTER);
+        top.add(actions, BorderLayout.SOUTH);
+        add(top, BorderLayout.NORTH);
+
+        Panel listing = new Panel(new BorderLayout(0, 5));
+        listing.setPreferredSize(new Dimension(285, 400));
+        listing.add(label("Images in this prefix"), BorderLayout.NORTH);
+        listing.add(images, BorderLayout.CENTER);
+        add(listing, BorderLayout.WEST);
+        add(preview, BorderLayout.CENTER);
+        add(status, BorderLayout.SOUTH);
+
+        connectButton.addActionListener(event -> connect());
+        upload.addActionListener(event -> chooseImages());
+        refresh.addActionListener(event -> loadPage(true));
+        moreButton.addActionListener(event -> loadPage(false));
+        download.addActionListener(event -> downloadSelected());
+        delete.addActionListener(event -> deleteSelected());
+        images.addItemListener(event -> previewSelected());
+        installDropTarget(this);
+        installDropTarget(preview);
+        installDropTarget(images);
+        addWindowListener(new WindowAdapter() {
+            @Override public void windowClosing(WindowEvent event) {
+                io.shutdownNow();
+                ObjectStorageClient previous = client;
+                if (previous != null) Thread.ofVirtual().start(previous::close);
+                dispose();
+            }
+        });
+        if (Boolean.parseBoolean(env("S3_AUTO_CONNECT", "false"))) {
+            EventQueue.invokeLater(this::connect);
+        }
+    }
+
+    private void connect() {
+        String url = endpoint.getText().trim();
+        String location = region.getText().trim();
+        String name = bucket.getText().trim();
+        String folder = normalizePrefix(prefix.getText());
+        String user = access.getText().trim();
+        String password = secret.getText();
+        boolean insecure = allowHttp.getState();
+        connectButton.setEnabled(false);
+        setStatus("Connecting...", false);
+        io.execute(() -> {
+            ObjectStorageClient candidate = null;
+            try {
+                ObjectStorageClientBuilder builder = new ObjectStorageClientBuilder()
+                    .endpoint(URI.create(url)).region(location).credentials(user, password)
+                    .timeout(Duration.ofSeconds(30));
+                if (insecure) builder.allowInsecureHttp();
+                candidate = builder.build();
+                candidate.listObjects(name, folder, null, 1);
+                ObjectStorageClient previous = client;
+                client = candidate;
+                activeBucket = name;
+                activePrefix = folder;
+                candidate = null;
+                if (previous != null) previous.close();
+                String service = "S3-compatible service";
+                try {
+                    Capabilities found = client.getCapabilities();
+                    if (found.service() == Capabilities.ServiceKind.OBJECTSTORE) {
+                        service = "ObjectStore" + (found.serviceVersion() == null ? "" : " " + found.serviceVersion());
+                    }
+                } catch (IOException ignored) {
+                    // Listing already established the connection; service detection is optional.
+                }
+                String connectedService = service;
+                EventQueue.invokeLater(() -> {
+                    connectButton.setEnabled(true);
+                    setStatus("Connected to " + connectedService + ".", false);
+                    loadPage(true);
+                });
+            } catch (Exception error) {
+                if (candidate != null) candidate.close();
+                showFailure("Connection failed", error);
+                EventQueue.invokeLater(() -> connectButton.setEnabled(true));
+            }
+        });
+    }
+
+    private void loadPage(boolean first) {
+        if (client == null) {
+            setStatus("Connect first.", true);
+            return;
+        }
+        if (!first && nextToken == null) return;
+        String token = first ? null : nextToken;
+        moreButton.setEnabled(false);
+        setStatus(first ? "Loading images..." : "Loading more images...", false);
+        io.execute(() -> {
+            try {
+                ObjectStorageClient.ObjectPage page = client.listObjects(activeBucket, activePrefix, token, 100);
+                EventQueue.invokeLater(() -> {
+                    if (first) {
+                        previewRevision.incrementAndGet();
+                        entries.clear();
+                        images.removeAll();
+                        preview.show(null, "Select an image to preview");
+                    }
+                    for (ObjectStorageClient.ObjectEntry entry : page.objects()) {
+                        if (!isImage(entry.key())) continue;
+                        entries.add(entry);
+                        String name = entry.key().substring(activePrefix.length());
+                        images.add(name + "  ·  " + sizeLabel(entry.size()));
+                    }
+                    nextToken = page.nextContinuationToken();
+                    moreButton.setEnabled(nextToken != null);
+                    setStatus(entries.size() + " image(s) loaded" +
+                        (nextToken == null ? "." : "; more available."), false);
+                });
+            } catch (Exception error) {
+                showFailure("Could not list images", error);
+                EventQueue.invokeLater(() -> moreButton.setEnabled(token != null));
+            }
+        });
+    }
+
+    private void chooseImages() {
+        FileDialog picker = new FileDialog(this, "Add images", FileDialog.LOAD);
+        picker.setMultipleMode(true);
+        picker.setVisible(true);
+        java.io.File[] selected = picker.getFiles();
+        if (selected.length > 0) uploadImages(List.of(selected));
+    }
+
+    private void uploadImages(List<java.io.File> files) {
+        if (client == null) {
+            setStatus("Connect first.", true);
+            return;
+        }
+        io.execute(() -> {
+            int uploaded = 0;
+            for (java.io.File file : files) {
+                try {
+                    Path path = file.toPath();
+                    String name = path.getFileName().toString();
+                    String type = contentType(name);
+                    if (type == null) throw new IOException("Unsupported image type");
+                    if (!Files.isRegularFile(path) || Files.size(path) > MAX_UPLOAD)
+                        throw new IOException("Image must be a file of at most 64 MiB");
+                    String key = activePrefix + UUID.randomUUID() + "-" + name;
+                    client.putObject(activeBucket, key, path, type);
+                    uploaded++;
+                    int completed = uploaded;
+                    EventQueue.invokeLater(() -> setStatus("Uploaded " + completed + " of " + files.size() + ".", false));
+                } catch (Exception error) {
+                    showFailure("Could not upload " + file.getName(), error);
+                }
+            }
+            if (uploaded > 0) EventQueue.invokeLater(() -> loadPage(true));
+        });
+    }
+
+    private void previewSelected() {
+        ObjectStorageClient.ObjectEntry entry = selectedEntry();
+        long revision = previewRevision.incrementAndGet();
+        if (entry == null) {
+            preview.show(null, "Select an image to preview");
+            return;
+        }
+        preview.show(null, "Loading preview...");
+        io.execute(() -> {
+            try (ObjectStorageClient.ObjectData data = client.getObject(activeBucket, entry.key())) {
+                if (data.length() > MAX_PREVIEW) throw new IOException("Preview exceeds 12 MiB");
+                byte[] bytes = data.body().readNBytes(MAX_PREVIEW + 1);
+                if (bytes.length > MAX_PREVIEW) throw new IOException("Preview exceeds 12 MiB");
+                BufferedImage image = decodePreview(bytes);
+                EventQueue.invokeLater(() -> {
+                    if (previewRevision.get() == revision) preview.show(image, null);
+                });
+            } catch (Exception error) {
+                EventQueue.invokeLater(() -> {
+                    if (previewRevision.get() == revision) preview.show(null, "Preview unavailable");
+                });
+                showFailure("Could not preview image", error);
+            }
+        });
+    }
+
+    private void downloadSelected() {
+        ObjectStorageClient.ObjectEntry entry = selectedEntry();
+        if (entry == null) {
+            setStatus("Select an image first.", true);
+            return;
+        }
+        FileDialog picker = new FileDialog(this, "Save image", FileDialog.SAVE);
+        picker.setFile(Path.of(entry.key()).getFileName().toString());
+        picker.setVisible(true);
+        if (picker.getFile() == null) return;
+        Path destination = Path.of(picker.getDirectory(), picker.getFile());
+        if (Files.exists(destination) && !confirm("Replace this file?", destination.toString())) return;
+        io.execute(() -> {
+            Path temporary = null;
+            try {
+                Path parent = destination.toAbsolutePath().getParent();
+                temporary = Files.createTempFile(parent, ".objectstore-image-", ".part");
+                try (ObjectStorageClient.ObjectData data = client.getObject(activeBucket, entry.key())) {
+                    Files.copy(data.body(), temporary, StandardCopyOption.REPLACE_EXISTING);
+                }
+                Files.move(temporary, destination, StandardCopyOption.REPLACE_EXISTING);
+                EventQueue.invokeLater(() -> setStatus("Saved " + destination.getFileName() + ".", false));
+            } catch (Exception error) {
+                showFailure("Download failed", error);
+            } finally {
+                if (temporary != null) {
+                    try { Files.deleteIfExists(temporary); } catch (IOException ignored) { }
+                }
+            }
+        });
+    }
+
+    private void deleteSelected() {
+        ObjectStorageClient.ObjectEntry entry = selectedEntry();
+        if (entry == null) {
+            setStatus("Select an image first.", true);
+            return;
+        }
+        if (!confirm("Delete this image?", entry.key())) return;
+        io.execute(() -> {
+            try {
+                client.deleteObject(activeBucket, entry.key());
+                EventQueue.invokeLater(() -> {
+                    previewRevision.incrementAndGet();
+                    loadPage(true);
+                });
+            } catch (Exception error) { showFailure("Delete failed", error); }
+        });
+    }
+
+    private ObjectStorageClient.ObjectEntry selectedEntry() {
+        int index = images.getSelectedIndex();
+        return index < 0 || index >= entries.size() ? null : entries.get(index);
+    }
+
+    private void installDropTarget(java.awt.Component target) {
+        new DropTarget(target, DnDConstants.ACTION_COPY, new DropTargetAdapter() {
+            @Override public void drop(DropTargetDropEvent event) {
+                if (!event.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) {
+                    event.rejectDrop();
+                    return;
+                }
+                try {
+                    event.acceptDrop(DnDConstants.ACTION_COPY);
+                    Object value = event.getTransferable().getTransferData(DataFlavor.javaFileListFlavor);
+                    List<?> dropped = (List<?>) value;
+                    List<java.io.File> files = new ArrayList<>();
+                    for (Object item : dropped) {
+                        if (item instanceof java.io.File file) files.add(file);
+                    }
+                    event.dropComplete(true);
+                    EventQueue.invokeLater(() -> uploadImages(files));
+                } catch (Exception error) {
+                    event.dropComplete(false);
+                    showFailure("Drop failed", error);
+                }
+            }
+        }, true);
+    }
+
+    private boolean confirm(String title, String detail) {
+        Dialog dialog = new Dialog(this, title, true);
+        dialog.setLayout(new BorderLayout(12, 12));
+        dialog.add(new Label(detail), BorderLayout.CENTER);
+        Panel buttons = new Panel(new FlowLayout(FlowLayout.RIGHT));
+        boolean[] accepted = { false };
+        Button cancel = new Button("Cancel");
+        Button proceed = new Button("Continue");
+        cancel.addActionListener(event -> dialog.dispose());
+        proceed.addActionListener(event -> {
+            accepted[0] = true;
+            dialog.dispose();
+        });
+        buttons.add(cancel);
+        buttons.add(proceed);
+        dialog.add(buttons, BorderLayout.SOUTH);
+        dialog.setSize(490, 120);
+        dialog.setLocationRelativeTo(this);
+        dialog.setVisible(true);
+        return accepted[0];
+    }
+
+    private void showFailure(String action, Exception error) {
+        EventQueue.invokeLater(() -> setStatus(action + ": " + error.getMessage(), true));
+    }
+
+    private void setStatus(String message, boolean failed) {
+        status.setForeground(failed ? new Color(245, 143, 157) : FOREGROUND);
+        status.setText(message);
+    }
+
+    private static Label label(String text) {
+        return new Label(text);
+    }
+
+    private static String normalizePrefix(String value) {
+        String cleaned = value.trim().replace('\\', '/');
+        while (cleaned.startsWith("/")) cleaned = cleaned.substring(1);
+        return cleaned.isEmpty() || cleaned.endsWith("/") ? cleaned : cleaned + "/";
+    }
+
+    private static String contentType(String name) {
+        String lower = name.toLowerCase(Locale.ROOT);
+        if (lower.endsWith(".png")) return "image/png";
+        if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
+        if (lower.endsWith(".gif")) return "image/gif";
+        if (lower.endsWith(".bmp")) return "image/bmp";
+        return null;
+    }
+
+    private static boolean isImage(String name) {
+        return contentType(name) != null;
+    }
+
+    private static String sizeLabel(long bytes) {
+        return bytes < 1024 ? bytes + " B" : String.format(Locale.ROOT, "%.1f KiB", bytes / 1024.0);
+    }
+
+    private static BufferedImage decodePreview(byte[] bytes) throws IOException {
+        try (ImageInputStream stream = new MemoryCacheImageInputStream(new ByteArrayInputStream(bytes))) {
+            var readers = ImageIO.getImageReaders(stream);
+            if (!readers.hasNext()) throw new IOException("Unsupported image data");
+            ImageReader reader = readers.next();
+            try {
+                reader.setInput(stream, true, true);
+                int width = reader.getWidth(0);
+                int height = reader.getHeight(0);
+                if (width < 1 || height < 1 || width > 16000 || height > 16000 ||
+                    (long) width * height > 40_000_000)
+                    throw new IOException("Image dimensions are too large for preview");
+                int step = Math.max(1, (Math.max(width, height) + 1199) / 1200);
+                var parameters = reader.getDefaultReadParam();
+                parameters.setSourceSubsampling(step, step, 0, 0);
+                return reader.read(0, parameters);
+            } finally { reader.dispose(); }
+        }
+    }
+
+    private static String env(String name, String fallback) {
+        String value = System.getenv(name);
+        return value == null ? fallback : value;
+    }
+
+    @SuppressWarnings("serial")
+    private static final class ImageCanvas extends Canvas {
+        private BufferedImage image;
+        private String message = "Select an image to preview";
+
+        private ImageCanvas() {
+            setBackground(new Color(21, 20, 29));
+            setForeground(FOREGROUND);
+            setFont(new Font(Font.SANS_SERIF, Font.PLAIN, 16));
+        }
+
+        private void show(BufferedImage value, String text) {
+            image = value;
+            message = text;
+            repaint();
+        }
+
+        @Override public void paint(Graphics graphics) {
+            int width = getWidth();
+            int height = getHeight();
+            if (image == null) {
+                graphics.setColor(FOREGROUND);
+                graphics.drawString(message, 20, Math.max(35, height / 2));
+                return;
+            }
+            double scale = Math.min((width - 24.0) / image.getWidth(),
+                (height - 24.0) / image.getHeight());
+            scale = Math.max(0.01, Math.min(scale, 1.0));
+            int drawWidth = (int) Math.round(image.getWidth() * scale);
+            int drawHeight = (int) Math.round(image.getHeight() * scale);
+            graphics.drawImage(image, (width - drawWidth) / 2, (height - drawHeight) / 2,
+                drawWidth, drawHeight, null);
+        }
+    }
+
+    public static void main(String[] args) {
+        EventQueue.invokeLater(() -> new ImageManager().setVisible(true));
+    }
+}

client/examples/awt-images/run-test.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+client_dir="$(cd "$(dirname "$0")/../.." && pwd)"
+project_dir="$(cd "$client_dir/.." && pwd)"
+container="objectstore-image-example"
+volume="objectstore-image-example-data"
+image="${OBJECTSTORE_TEST_IMAGE:-objectstore-image-example:local}"
+
+if ! docker container inspect "$container" >/dev/null 2>&1; then
+  if ! docker image inspect "$image" >/dev/null 2>&1; then
+    docker build --tag "$image" "$project_dir"
+  fi
+  export S3_ACCESS_KEY="ImageExampleTest1"
+  export S3_SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_hex(32))')"
+  docker volume create "$volume" >/dev/null
+  docker run --detach --name "$container" \
+    --publish 127.0.0.1:9002:9000 \
+    --volume "$volume:/data" \
+    --env S3_ACCESS_KEY --env S3_SECRET_KEY \
+    --env S3_BUCKET=photos --env S3_REGION=us-east-1 \
+    --env MAX_OBJECT_BYTES=67108864 --env MAX_TOTAL_BYTES=1073741824 \
+    "$image" >/dev/null
+else
+  while IFS='=' read -r key value; do
+    case "$key" in
+      S3_ACCESS_KEY) export S3_ACCESS_KEY="$value" ;;
+      S3_SECRET_KEY) export S3_SECRET_KEY="$value" ;;
+    esac
+  done < <(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$container")
+  if [[ "$(docker inspect --format '{{.State.Running}}' "$container")" != true ]]; then
+    docker start "$container" >/dev/null
+  fi
+fi
+
+for attempt in {1..40}; do
+  if curl --silent --fail --output /dev/null http://127.0.0.1:9002/health; then
+    break
+  fi
+  sleep 0.25
+done
+if ! curl --silent --fail --output /dev/null http://127.0.0.1:9002/health; then
+  echo "The isolated ObjectStore container did not become healthy; check docker logs $container." >&2
+  exit 1
+fi
+export S3_ENDPOINT=http://127.0.0.1:9002
+export S3_REGION=us-east-1
+export S3_BUCKET=photos
+export S3_ALLOW_HTTP=true
+export S3_AUTO_CONNECT=true
+exec "$client_dir/examples/awt-images/run.sh"

client/examples/awt-images/run.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+client_dir="$(cd "$(dirname "$0")/../.." && pwd)"
+if [[ ! -f "$client_dir/dist/objectstore-client.jar" ]]; then
+  bash "$client_dir/scripts/build.sh"
+fi
+classes="$client_dir/dist/examples/awt-images"
+mkdir -p "$classes"
+javac --release 21 -cp "$client_dir/dist/objectstore-client.jar" \
+  -d "$classes" "$client_dir/examples/awt-images/ImageManager.java"
+exec java -cp "$classes:$client_dir/dist/objectstore-client.jar" ImageManager

client/scripts/build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+rm -rf dist/classes
+mkdir -p dist/classes
+find src/main/java -name '*.java' -print0 |
+  xargs -0 javac --release 21 -d dist/classes
+mkdir -p dist/classes/META-INF
+cp ../LICENSE dist/classes/META-INF/LICENSE
+jar --create --file dist/objectstore-client.jar -C dist/classes .
+javadoc --release 21 -quiet -Xdoclint:reference,syntax,html -d dist/javadoc \
+  $(find src/main/java -name '*.java' -print)
+echo "Built dist/objectstore-client.jar"

client/scripts/test.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+build_dir="$(mktemp -d)"
+trap 'rm -rf "$build_dir"' EXIT
+find src/main/java src/test/java -name '*.java' -print0 |
+  xargs -0 javac --release 21 -d "$build_dir"
+java -cp "$build_dir" cloud.lunarsky.objectstore.client.ClientTest
+java -cp "$build_dir" cloud.lunarsky.objectstore.client.MultipartClientTest

client/src/main/java/cloud/lunarsky/objectstore/client/AclPolicy.java

@@ -0,0 +1,27 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.util.List;
+import java.util.Objects;
+
+/** A bucket or object ACL. This is not an effective-permissions calculation. */
+public record AclPolicy(String ownerId, List<Grant> grants) {
+    /** A canonical user or predefined S3 group. */
+    public enum GranteeType { CANONICAL_USER, GROUP }
+
+    /** An S3 ACL permission. */
+    public enum Permission { READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL }
+
+    /** One ACL grant to a canonical user ID or S3 group URI. */
+    public record Grant(GranteeType type, String grantee, Permission permission) {
+        public Grant {
+            Objects.requireNonNull(type, "type");
+            if (grantee == null || grantee.isBlank()) throw new IllegalArgumentException("grantee is required");
+            Objects.requireNonNull(permission, "permission");
+        }
+    }
+
+    public AclPolicy {
+        if (ownerId == null || ownerId.isBlank()) throw new IllegalArgumentException("owner ID is required");
+        grants = List.copyOf(grants);
+    }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/Capabilities.java

@@ -0,0 +1,43 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/** Reported or observed operation support, separate from the caller's authorization. */
+public record Capabilities(ServiceKind service, Map<String, Support> operations, Map<String, Long> limits,
+                           String serviceVersion, String storageMode, boolean completeOperationInventory) {
+    private static final Set<String> KNOWN_OPERATIONS = Set.of(
+        "ListBuckets", "CreateBucket", "HeadBucket", "DeleteBucket", "ListObjectsV2",
+        "PutObject", "GetObject", "HeadObject", "DeleteObject", "CopyObject",
+        "GetObjectTagging", "PutObjectTagging", "DeleteObjectTagging",
+        "CreateMultipartUpload", "UploadPart", "ListParts", "CompleteMultipartUpload",
+        "AbortMultipartUpload", "ListMultipartUploads", "GetBucketVersioning",
+        "PutBucketVersioning", "ListObjectVersions", "GetBucketAcl", "PutBucketAcl",
+        "GetObjectAcl", "PutObjectAcl");
+
+    /** Identifies a self-reported ObjectStore service or an unrecognized S3-compatible service. */
+    public enum ServiceKind { OBJECTSTORE, UNKNOWN_S3 }
+
+    /** Three-state feature support; a generic S3 server cannot be inferred from its hostname. */
+    public enum Support { SUPPORTED, UNSUPPORTED, UNKNOWN }
+
+    public Capabilities {
+        Objects.requireNonNull(service, "service");
+        operations = Map.copyOf(operations);
+        limits = Map.copyOf(limits);
+    }
+
+    /** Constructs a result without a server manifest or configured limits. */
+    public Capabilities(ServiceKind service, Map<String, Support> operations) {
+        this(service, operations, Map.of(), null, null, false);
+    }
+
+    /** Returns support for a named operation. A missing manifest or denied probe stays UNKNOWN. */
+    public Support support(String operation) {
+        Support observed = operations.get(operation);
+        if (observed != null) return observed;
+        if (completeOperationInventory && KNOWN_OPERATIONS.contains(operation)) return Support.UNSUPPORTED;
+        return Support.UNKNOWN;
+    }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/Json.java

@@ -0,0 +1,208 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+final class Json {
+    private Json() { }
+
+    static Object parse(byte[] bytes) throws ProtocolException {
+        String source;
+        try { source = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes)).toString(); }
+        catch (CharacterCodingException e) { throw new ProtocolException("Capability JSON is not UTF-8", e); }
+        Parser parser = new Parser(source);
+        Object value = parser.value(0);
+        parser.space();
+        if (parser.index != source.length()) throw new ProtocolException("Trailing capability JSON data");
+        return value;
+    }
+
+    @SuppressWarnings("unchecked")
+    static Map<String, Object> object(Object value, String name) throws ProtocolException {
+        if (!(value instanceof Map<?, ?>)) throw new ProtocolException(name + " must be an object");
+        return (Map<String, Object>) value;
+    }
+
+    static List<?> array(Object value, String name) throws ProtocolException {
+        if (!(value instanceof List<?> list)) throw new ProtocolException(name + " must be an array");
+        return list;
+    }
+
+    static String string(Object value, String name) throws ProtocolException {
+        if (!(value instanceof String text) || text.isBlank())
+            throw new ProtocolException(name + " must be a nonempty string");
+        return text;
+    }
+
+    static long integer(Object value, String name) throws ProtocolException {
+        if (!(value instanceof BigDecimal number)) throw new ProtocolException(name + " must be an integer");
+        try { return number.longValueExact(); }
+        catch (ArithmeticException e) { throw new ProtocolException(name + " is out of range", e); }
+    }
+
+    private static final class Parser {
+        private final String source;
+        private int index;
+
+        private Parser(String source) { this.source = source; }
+
+        private void space() {
+            while (index < source.length() && (source.charAt(index) == ' ' || source.charAt(index) == '\n' ||
+                source.charAt(index) == '\r' || source.charAt(index) == '\t')) index++;
+        }
+
+        private Object value(int depth) throws ProtocolException {
+            if (depth > 16) throw new ProtocolException("Capability JSON is too deeply nested");
+            space();
+            if (index >= source.length()) throw new ProtocolException("Incomplete capability JSON");
+            return switch (source.charAt(index)) {
+                case '{' -> object(depth + 1);
+                case '[' -> array(depth + 1);
+                case '"' -> string();
+                case 't' -> literal("true", Boolean.TRUE);
+                case 'f' -> literal("false", Boolean.FALSE);
+                case 'n' -> literal("null", null);
+                default -> number();
+            };
+        }
+
+        private Map<String, Object> object(int depth) throws ProtocolException {
+            index++;
+            space();
+            Map<String, Object> result = new HashMap<>();
+            if (take('}')) return result;
+            do {
+                space();
+                if (index >= source.length() || source.charAt(index) != '"')
+                    throw new ProtocolException("Capability JSON object key is missing");
+                String key = string();
+                space();
+                require(':');
+                if (result.containsKey(key)) throw new ProtocolException("Duplicate capability JSON key");
+                result.put(key, value(depth));
+                if (result.size() > 256) throw new ProtocolException("Capability JSON object is too large");
+                space();
+                if (take('}')) return result;
+                require(',');
+            } while (true);
+        }
+
+        private List<Object> array(int depth) throws ProtocolException {
+            index++;
+            space();
+            List<Object> result = new ArrayList<>();
+            if (take(']')) return result;
+            do {
+                result.add(value(depth));
+                if (result.size() > 2048) throw new ProtocolException("Capability JSON array is too large");
+                space();
+                if (take(']')) return result;
+                require(',');
+            } while (true);
+        }
+
+        private String string() throws ProtocolException {
+            index++;
+            StringBuilder result = new StringBuilder();
+            while (index < source.length()) {
+                char current = source.charAt(index++);
+                if (current == '"') {
+                    for (int i = 0; i < result.length(); i++) {
+                        char unit = result.charAt(i);
+                        if (Character.isHighSurrogate(unit)) {
+                            if (++i >= result.length() || !Character.isLowSurrogate(result.charAt(i)))
+                                throw new ProtocolException("Invalid JSON Unicode surrogate");
+                        } else if (Character.isLowSurrogate(unit))
+                            throw new ProtocolException("Invalid JSON Unicode surrogate");
+                    }
+                    return result.toString();
+                }
+                if (current < 0x20) throw new ProtocolException("Unescaped control character in JSON string");
+                if (current != '\\') {
+                    result.append(current);
+                    continue;
+                }
+                if (index >= source.length()) throw new ProtocolException("Incomplete JSON escape");
+                char escaped = source.charAt(index++);
+                switch (escaped) {
+                    case '"', '\\', '/' -> result.append(escaped);
+                    case 'b' -> result.append('\b');
+                    case 'f' -> result.append('\f');
+                    case 'n' -> result.append('\n');
+                    case 'r' -> result.append('\r');
+                    case 't' -> result.append('\t');
+                    case 'u' -> {
+                        if (index + 4 > source.length()) throw new ProtocolException("Incomplete Unicode escape");
+                        int unit = 0;
+                        for (int i = 0; i < 4; i++) {
+                            int digit = Character.digit(source.charAt(index++), 16);
+                            if (digit < 0) throw new ProtocolException("Invalid Unicode escape");
+                            unit = unit * 16 + digit;
+                        }
+                        result.append((char) unit);
+                    }
+                    default -> throw new ProtocolException("Invalid JSON escape");
+                }
+            }
+            throw new ProtocolException("Unterminated JSON string");
+        }
+
+        private Object literal(String expected, Object value) throws ProtocolException {
+            if (!source.startsWith(expected, index)) throw new ProtocolException("Invalid JSON literal");
+            index += expected.length();
+            return value;
+        }
+
+        private BigDecimal number() throws ProtocolException {
+            int start = index;
+            if (take('-') && index >= source.length()) throw new ProtocolException("Invalid JSON number");
+            integerDigits();
+            if (take('.')) requireDigits("Invalid JSON fraction");
+            if (take('e') || take('E')) {
+                if (!take('+')) take('-');
+                requireDigits("Invalid JSON exponent");
+            }
+            try { return new BigDecimal(source.substring(start, index)); }
+            catch (NumberFormatException e) { throw new ProtocolException("Invalid JSON number", e); }
+        }
+
+        private void integerDigits() throws ProtocolException {
+            if (take('0')) {
+                if (index < source.length() && Character.isDigit(source.charAt(index)))
+                    throw new ProtocolException("Invalid JSON number");
+            } else {
+                if (index >= source.length() || source.charAt(index) < '1' || source.charAt(index) > '9')
+                    throw new ProtocolException("Invalid JSON number");
+                scanDigits();
+            }
+        }
+
+        private void requireDigits(String message) throws ProtocolException {
+            int first = index;
+            scanDigits();
+            if (first == index) throw new ProtocolException(message);
+        }
+
+        private void scanDigits() {
+            while (index < source.length() && source.charAt(index) >= '0' && source.charAt(index) <= '9') index++;
+        }
+
+        private boolean take(char value) {
+            if (index < source.length() && source.charAt(index) == value) {
+                index++;
+                return true;
+            }
+            return false;
+        }
+
+        private void require(char value) throws ProtocolException {
+            if (!take(value)) throw new ProtocolException("Expected '" + value + "' in capability JSON");
+        }
+    }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClient.java

@@ -0,0 +1,633 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URLDecoder;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.w3c.dom.Element;
+
+/** A synchronous, path-style S3 client. Instances are safe to share between threads. */
+public final class ObjectStorageClient implements AutoCloseable {
+    private static final int MAX_XML = 4 * 1024 * 1024;
+    private static final int MAX_ERROR = 64 * 1024;
+    private static final String EMPTY_HASH = SigV4.hash(new byte[0]);
+    private final URI endpoint;
+    private final String region;
+    private final String accessKey;
+    private final String secretKey;
+    private final Duration timeout;
+    private final Clock clock;
+    private final HttpClient http;
+    private final Set<String> observedOperations = ConcurrentHashMap.newKeySet();
+
+    ObjectStorageClient(URI endpoint, String region, String accessKey, String secretKey,
+                        Duration timeout, Clock clock) {
+        this.endpoint = endpoint;
+        this.region = region;
+        this.accessKey = accessKey;
+        this.secretKey = secretKey;
+        this.timeout = timeout;
+        this.clock = clock;
+        this.http = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER)
+            .connectTimeout(timeout).version(HttpClient.Version.HTTP_1_1).build();
+    }
+
+    /** Returns one open object stream. The caller must close the returned result. */
+    public ObjectData getObject(String bucket, String key) throws IOException {
+        HttpResponse<InputStream> response = send("GET", objectPath(bucket, key), Map.of(), Map.of(),
+            HttpRequest.BodyPublishers.noBody(), EMPTY_HASH);
+        if (!successful(response)) {
+            try (InputStream ignored = response.body()) { throw failure(response); }
+        }
+        observedOperations.add("GetObject");
+        return new ObjectData(response.body(), length(response), response.headers().firstValue("content-type").orElse(null),
+            response.headers().firstValue("etag").orElse(null));
+    }
+
+    /** Returns object headers without downloading content. */
+    public ObjectMetadata headObject(String bucket, String key) throws IOException {
+        HttpResponse<InputStream> response = send("HEAD", objectPath(bucket, key), Map.of(), Map.of(),
+            HttpRequest.BodyPublishers.noBody(), EMPTY_HASH);
+        try (InputStream ignored = response.body()) {
+            if (!successful(response)) throw failure(response);
+            observedOperations.add("HeadObject");
+            return new ObjectMetadata(length(response), response.headers().firstValue("content-type").orElse(null),
+                response.headers().firstValue("etag").orElse(null));
+        }
+    }
+
+    /** Uploads an immutable byte array as one object. No automatic write retry is performed. */
+    public void putObject(String bucket, String key, byte[] content, String contentType) throws IOException {
+        Objects.requireNonNull(content, "content");
+        put(bucket, key, HttpRequest.BodyPublishers.ofByteArray(content), SigV4.hash(content), contentType);
+    }
+
+    /** Uploads a file. Do not change the file between hashing and completion of the upload. */
+    public void putObject(String bucket, String key, Path file, String contentType) throws IOException {
+        Objects.requireNonNull(file, "file");
+        String hash;
+        try (InputStream input = Files.newInputStream(file)) {
+            MessageDigest digest = MessageDigest.getInstance("SHA-256");
+            byte[] buffer = new byte[65536];
+            for (int count; (count = input.read(buffer)) >= 0; ) digest.update(buffer, 0, count);
+            hash = java.util.HexFormat.of().formatHex(digest.digest());
+        } catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
+        put(bucket, key, HttpRequest.BodyPublishers.ofFile(file), hash, contentType);
+    }
+
+    private void put(String bucket, String key, HttpRequest.BodyPublisher body, String hash,
+                     String contentType) throws IOException {
+        Map<String, String> headers = contentType == null ? Map.of() : Map.of("content-type", contentType);
+        HttpResponse<InputStream> response = send("PUT", objectPath(bucket, key), Map.of(), headers, body, hash);
+        try (InputStream ignored = response.body()) {
+            if (!successful(response)) throw failure(response);
+            observedOperations.add("PutObject");
+        }
+    }
+
+    /** Deletes the current object. A versioned backend may create a delete marker. */
+    public void deleteObject(String bucket, String key) throws IOException {
+        HttpResponse<InputStream> response = send("DELETE", objectPath(bucket, key), Map.of(), Map.of(),
+            HttpRequest.BodyPublishers.noBody(), EMPTY_HASH);
+        try (InputStream ignored = response.body()) {
+            if (!successful(response)) throw failure(response);
+            observedOperations.add("DeleteObject");
+        }
+    }
+
+    /** Lists one page of objects using S3 ListObjectsV2. Pass the returned token to get the next page. */
+    public ObjectPage listObjects(String bucket, String prefix, String continuationToken, int maxKeys)
+        throws IOException {
+        if (maxKeys < 1 || maxKeys > 1000) throw new IllegalArgumentException("maxKeys must be 1..1000");
+        Map<String, String> query = new HashMap<>();
+        query.put("list-type", "2");
+        query.put("encoding-type", "url");
+        query.put("max-keys", Integer.toString(maxKeys));
+        if (prefix != null) query.put("prefix", prefix);
+        if (continuationToken != null) query.put("continuation-token", continuationToken);
+        Element root = xml("GET", bucketPath(bucket), query);
+        requireRoot(root, "ListBucketResult");
+        observedOperations.add("ListObjectsV2");
+        List<ObjectEntry> entries = new ArrayList<>();
+        var nodes = Xml.descendants(root, "Contents");
+        for (int i = 0; i < nodes.getLength(); i++) {
+            Element entry = (Element) nodes.item(i);
+            String key = Xml.text(entry, "Key");
+            String size = Xml.text(entry, "Size");
+            if (key == null || size == null) throw new ProtocolException("Incomplete object listing");
+            try {
+                entries.add(new ObjectEntry(URLDecoder.decode(key.replace("+", "%2B"), StandardCharsets.UTF_8),
+                    Long.parseLong(size), Xml.text(entry, "ETag")));
+            } catch (IllegalArgumentException e) { throw new ProtocolException("Invalid object listing entry", e); }
+        }
+        String next = Xml.text(root, "NextContinuationToken");
+        if ("true".equalsIgnoreCase(Xml.text(root, "IsTruncated")) && (next == null || next.isEmpty()))
+            throw new ProtocolException("Truncated listing has no continuation token");
+        return new ObjectPage(entries, next);
+    }
+
+    /** Starts a standard S3 multipart upload. Keep the returned upload ID to resume or abort later. */
+    public MultipartUpload createMultipartUpload(String bucket, String key, String contentType) throws IOException {
+        String path = objectPath(bucket, key);
+        Map<String, String> headers = contentType == null ? Map.of() : Map.of("content-type", contentType);
+        HttpResponse<InputStream> response = send("POST", path, Map.of("uploads", ""), headers,
+            HttpRequest.BodyPublishers.noBody(), EMPTY_HASH);
+        Element root = responseXml(response, "InitiateMultipartUploadResult");
+        String id = Xml.text(root, "UploadId");
+        if (id == null || id.isBlank()) throw new ProtocolException("Multipart initiation has no upload ID");
+        observedOperations.add("CreateMultipartUpload");
+        return new MultipartUpload(bucket, key, id);
+    }
+
+    /** Uploads one part. The ETag in the result must be supplied when completing the upload. */
+    public Part uploadPart(MultipartUpload upload, int partNumber, byte[] content) throws IOException {
+        Objects.requireNonNull(upload, "upload");
+        Objects.requireNonNull(content, "content");
+        checkPartNumber(partNumber);
+        HttpResponse<InputStream> response = send("PUT", objectPath(upload.bucket(), upload.key()),
+            Map.of("uploadId", upload.uploadId(), "partNumber", Integer.toString(partNumber)), Map.of(),
+            HttpRequest.BodyPublishers.ofByteArray(content), SigV4.hash(content));
+        try (InputStream ignored = response.body()) {
+            if (!successful(response)) throw failure(response);
+            String etag = response.headers().firstValue("etag").orElse(null);
+            if (etag == null || etag.isBlank()) throw new ProtocolException("Part upload has no ETag");
+            observedOperations.add("UploadPart");
+            return new Part(partNumber, etag, content.length);
+        }
+    }
+
+    /**
+     * Uploads an immutable file as sequential parts without completing it. The caller owns the
+     * upload session and can retry, list parts, complete, or abort it after any failure.
+     * Each part is held in memory once; partSize must be 5 to 128 MiB.
+     */
+    public List<Part> uploadFileParts(MultipartUpload upload, Path file, int partSize) throws IOException {
+        Objects.requireNonNull(upload, "upload");
+        Objects.requireNonNull(file, "file");
+        if (partSize < 5 * 1024 * 1024 || partSize > 128 * 1024 * 1024)
+            throw new IllegalArgumentException("partSize must be 5..128 MiB");
+        long size = Files.size(file);
+        if (size < 1 || (size + partSize - 1L) / partSize > 10000)
+            throw new IllegalArgumentException("file requires 1..10000 parts");
+        List<Part> parts = new ArrayList<>();
+        try (InputStream input = Files.newInputStream(file)) {
+            long remaining = size;
+            for (int number = 1; remaining > 0; number++) {
+                int length = (int) Math.min(remaining, partSize);
+                byte[] bytes = input.readNBytes(length);
+                if (bytes.length != length) throw new IOException("File changed during multipart upload");
+                parts.add(uploadPart(upload, number, bytes));
+                remaining -= length;
+            }
+            if (input.read() != -1) throw new IOException("File changed during multipart upload");
+        }
+        return List.copyOf(parts);
+    }
+
+    /** Lists one page of uploaded parts for resume or verification. */
+    public PartPage listParts(MultipartUpload upload, int partNumberMarker, int maxParts) throws IOException {
+        Objects.requireNonNull(upload, "upload");
+        if (partNumberMarker < 0 || partNumberMarker > 10000 || maxParts < 1 || maxParts > 1000)
+            throw new IllegalArgumentException("invalid part marker or page size");
+        Element root = xml("GET", objectPath(upload.bucket(), upload.key()),
+            Map.of("uploadId", upload.uploadId(), "part-number-marker", Integer.toString(partNumberMarker),
+                "max-parts", Integer.toString(maxParts)));
+        requireRoot(root, "ListPartsResult");
+        observedOperations.add("ListParts");
+        List<Part> parts = new ArrayList<>();
+        var nodes = Xml.descendants(root, "Part");
+        for (int i = 0; i < nodes.getLength(); i++) {
+            Element entry = (Element) nodes.item(i);
+            try {
+                parts.add(new Part(Integer.parseInt(requiredText(entry, "PartNumber")),
+                    requiredText(entry, "ETag"), Long.parseLong(requiredText(entry, "Size"))));
+            } catch (NumberFormatException e) { throw new ProtocolException("Invalid part listing entry", e); }
+        }
+        boolean truncated = Boolean.parseBoolean(Xml.text(root, "IsTruncated"));
+        int next = partNumberMarker;
+        if (truncated) {
+            try { next = Integer.parseInt(requiredText(root, "NextPartNumberMarker")); }
+            catch (NumberFormatException e) { throw new ProtocolException("Invalid next part marker", e); }
+            if (next <= partNumberMarker) throw new ProtocolException("Part listing did not advance");
+        }
+        return new PartPage(parts, truncated, next);
+    }
+
+    /** Completes the upload using the exact part numbers and ETags returned by the service. */
+    public String completeMultipartUpload(MultipartUpload upload, List<Part> parts) throws IOException {
+        Objects.requireNonNull(upload, "upload");
+        Objects.requireNonNull(parts, "parts");
+        if (parts.isEmpty() || parts.size() > 10000) throw new IllegalArgumentException("1..10000 parts required");
+        List<Part> ordered = new ArrayList<>(parts);
+        ordered.sort(Comparator.comparingInt(Part::number));
+        StringBuilder xml = new StringBuilder("<CompleteMultipartUpload>");
+        int previous = 0;
+        for (Part part : ordered) {
+            Objects.requireNonNull(part, "part");
+            checkPartNumber(part.number());
+            if (part.number() == previous) throw new IllegalArgumentException("duplicate part number");
+            previous = part.number();
+            xml.append("<Part><PartNumber>").append(part.number()).append("</PartNumber><ETag>")
+                .append(Xml.escape(part.etag())).append("</ETag></Part>");
+        }
+        byte[] bytes = xml.append("</CompleteMultipartUpload>").toString().getBytes(StandardCharsets.UTF_8);
+        HttpResponse<InputStream> response = send("POST", objectPath(upload.bucket(), upload.key()),
+            Map.of("uploadId", upload.uploadId()), Map.of("content-type", "application/xml"),
+            HttpRequest.BodyPublishers.ofByteArray(bytes), SigV4.hash(bytes));
+        Element root = responseXml(response, "CompleteMultipartUploadResult");
+        observedOperations.add("CompleteMultipartUpload");
+        return requiredText(root, "ETag");
+    }
+
+    /** Aborts an unfinished upload. A completed upload cannot be aborted. */
+    public void abortMultipartUpload(MultipartUpload upload) throws IOException {
+        Objects.requireNonNull(upload, "upload");
+        HttpResponse<InputStream> response = send("DELETE", objectPath(upload.bucket(), upload.key()),
+            Map.of("uploadId", upload.uploadId()), Map.of(), HttpRequest.BodyPublishers.noBody(), EMPTY_HASH);
+        try (InputStream ignored = response.body()) {
+            if (!successful(response)) throw failure(response);
+            observedOperations.add("AbortMultipartUpload");
+        }
+    }
+
+    /** Lists one page of unfinished uploads in a bucket. */
+    public MultipartUploadPage listMultipartUploads(String bucket, String prefix, String keyMarker,
+                                                    String uploadIdMarker, int maxUploads) throws IOException {
+        if (maxUploads < 1 || maxUploads > 1000) throw new IllegalArgumentException("maxUploads must be 1..1000");
+        if (uploadIdMarker != null && keyMarker == null)
+            throw new IllegalArgumentException("upload ID marker requires key marker");
+        Map<String, String> query = new HashMap<>();
+        query.put("uploads", "");
+        query.put("max-uploads", Integer.toString(maxUploads));
+        if (prefix != null) query.put("prefix", prefix);
+        if (keyMarker != null) query.put("key-marker", keyMarker);
+        if (uploadIdMarker != null) query.put("upload-id-marker", uploadIdMarker);
+        Element root = xml("GET", bucketPath(bucket), query);
+        requireRoot(root, "ListMultipartUploadsResult");
+        observedOperations.add("ListMultipartUploads");
+        List<MultipartUpload> uploads = new ArrayList<>();
+        var nodes = Xml.descendants(root, "Upload");
+        for (int i = 0; i < nodes.getLength(); i++) {
+            Element entry = (Element) nodes.item(i);
+            uploads.add(new MultipartUpload(bucket, requiredText(entry, "Key"), requiredText(entry, "UploadId")));
+        }
+        boolean truncated = Boolean.parseBoolean(Xml.text(root, "IsTruncated"));
+        String nextKey = Xml.text(root, "NextKeyMarker");
+        String nextId = Xml.text(root, "NextUploadIdMarker");
+        if (truncated && (nextKey == null || nextId == null))
+            throw new ProtocolException("Truncated upload listing has no markers");
+        return new MultipartUploadPage(uploads, truncated, nextKey, nextId);
+    }
+
+    private static String requiredText(Element element, String name) throws ProtocolException {
+        String value = Xml.text(element, name);
+        if (value == null || value.isBlank()) throw new ProtocolException("Multipart response missing " + name);
+        return value;
+    }
+
+    private static void checkPartNumber(int number) {
+        if (number < 1 || number > 10000) throw new IllegalArgumentException("part number must be 1..10000");
+    }
+
+    /** Gets a bucket ACL. The backend may reject ACL operations when ACLs are disabled. */
+    public AclPolicy getBucketAcl(String bucket) throws IOException {
+        AclPolicy policy = readAcl(bucketPath(bucket));
+        observedOperations.add("GetBucketAcl");
+        return policy;
+    }
+
+    /** Gets an object ACL. This does not calculate permissions from policies or other grants. */
+    public AclPolicy getObjectAcl(String bucket, String key) throws IOException {
+        AclPolicy policy = readAcl(objectPath(bucket, key));
+        observedOperations.add("GetObjectAcl");
+        return policy;
+    }
+
+    /** Replaces a bucket ACL. No automatic retry is performed. */
+    public void putBucketAcl(String bucket, AclPolicy policy) throws IOException {
+        writeAcl(bucketPath(bucket), policy);
+        observedOperations.add("PutBucketAcl");
+    }
+
+    /** Replaces an object ACL. No automatic retry is performed. */
+    public void putObjectAcl(String bucket, String key, AclPolicy policy) throws IOException {
+        writeAcl(objectPath(bucket, key), policy);
+        observedOperations.add("PutObjectAcl");
+    }
+
+    private AclPolicy readAcl(String path) throws IOException {
+        Element root = xml("GET", path, Map.of("acl", ""));
+        requireRoot(root, "AccessControlPolicy");
+        Element owner = Xml.child(root, "Owner");
+        String ownerId = owner == null ? null : Xml.text(owner, "ID");
+        if (ownerId == null || ownerId.isBlank()) throw new ProtocolException("ACL response has no owner ID");
+        List<AclPolicy.Grant> grants = new ArrayList<>();
+        var nodes = Xml.descendants(root, "Grant");
+        for (int i = 0; i < nodes.getLength(); i++) {
+            Element grant = (Element) nodes.item(i);
+            Element grantee = Xml.child(grant, "Grantee");
+            if (grantee == null) throw new ProtocolException("ACL grant has no grantee");
+            String type = grantee.getAttributeNS("http://www.w3.org/2001/XMLSchema-instance", "type");
+            if (type.isEmpty()) type = grantee.getAttribute("xsi:type");
+            AclPolicy.GranteeType kind = switch (type) {
+                case "CanonicalUser" -> AclPolicy.GranteeType.CANONICAL_USER;
+                case "Group" -> AclPolicy.GranteeType.GROUP;
+                default -> throw new ProtocolException("Unsupported ACL grantee type");
+            };
+            String id = Xml.text(grantee, kind == AclPolicy.GranteeType.GROUP ? "URI" : "ID");
+            String permission = Xml.text(grant, "Permission");
+            if (id == null || permission == null) throw new ProtocolException("Incomplete ACL grant");
+            try { grants.add(new AclPolicy.Grant(kind, id, AclPolicy.Permission.valueOf(permission))); }
+            catch (IllegalArgumentException e) { throw new ProtocolException("Unsupported ACL permission", e); }
+        }
+        return new AclPolicy(ownerId, grants);
+    }
+
+    private void writeAcl(String path, AclPolicy policy) throws IOException {
+        Objects.requireNonNull(policy, "policy");
+        StringBuilder xml = new StringBuilder("<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\" ")
+            .append("xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><Owner><ID>")
+            .append(Xml.escape(policy.ownerId())).append("</ID></Owner><AccessControlList>");
+        for (AclPolicy.Grant grant : policy.grants()) {
+            boolean group = grant.type() == AclPolicy.GranteeType.GROUP;
+            xml.append("<Grant><Grantee xsi:type=\"").append(group ? "Group" : "CanonicalUser")
+                .append("\"><").append(group ? "URI" : "ID").append('>')
+                .append(Xml.escape(grant.grantee())).append("</").append(group ? "URI" : "ID")
+                .append("></Grantee><Permission>").append(grant.permission()).append("</Permission></Grant>");
+        }
+        xml.append("</AccessControlList></AccessControlPolicy>");
+        byte[] bytes = xml.toString().getBytes(StandardCharsets.UTF_8);
+        String md5;
+        try { md5 = Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(bytes)); }
+        catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
+        HttpResponse<InputStream> response = send("PUT", path, Map.of("acl", ""),
+            Map.of("content-type", "application/xml", "content-md5", md5),
+            HttpRequest.BodyPublishers.ofByteArray(bytes), SigV4.hash(bytes));
+        try (InputStream ignored = response.body()) {
+            if (!successful(response)) throw failure(response);
+        }
+    }
+
+    /**
+     * Reads ObjectStore's signed manifest when present. Other S3 endpoints report only operations
+     * observed succeeding on this client. A denied or missing manifest does not imply a feature is
+     * unsupported. Service identity from /health is self-reported, not cryptographic attestation.
+     */
+    public Capabilities getCapabilities() throws IOException {
+        HttpResponse<InputStream> response = send("GET", "/_objectstore/capabilities", Map.of(), Map.of(),
+            HttpRequest.BodyPublishers.noBody(), EMPTY_HASH);
+        ProtocolException invalidManifest = null;
+        try (InputStream body = response.body()) {
+            if (successful(response)) {
+                try { return withObserved(parseManifest(readLimited(body, 64 * 1024))); }
+                catch (ProtocolException e) { invalidManifest = e; }
+            }
+        }
+        boolean objectStore = probeHealth();
+        if (invalidManifest != null && objectStore) throw invalidManifest;
+        return withObserved(new Capabilities(objectStore ? Capabilities.ServiceKind.OBJECTSTORE :
+            Capabilities.ServiceKind.UNKNOWN_S3, Map.of()));
+    }
+
+    /**
+     * Opts into read-only probes on a known bucket and, optionally, an existing object key.
+     * Successful calls establish support for this principal. S3 error responses leave support
+     * UNKNOWN; transport and malformed-response errors are still reported to the caller.
+     */
+    public Capabilities probeReadOnlyCapabilities(String bucket, String existingObjectKey) throws IOException {
+        Capabilities base = getCapabilities();
+        probe(base, "ListObjectsV2", () -> listObjects(bucket, null, null, 1));
+        probe(base, "ListMultipartUploads", () -> listMultipartUploads(bucket, null, null, null, 1));
+        probe(base, "GetBucketAcl", () -> getBucketAcl(bucket));
+        if (existingObjectKey != null) {
+            probe(base, "HeadObject", () -> headObject(bucket, existingObjectKey));
+            probe(base, "GetObjectAcl", () -> getObjectAcl(bucket, existingObjectKey));
+        }
+        return withObserved(base);
+    }
+
+    private void probe(Capabilities base, String operation, Probe call) throws IOException {
+        if (base.support(operation) == Capabilities.Support.UNSUPPORTED) return;
+        try { call.run(); }
+        catch (ObjectStorageException ignored) { }
+    }
+
+    @FunctionalInterface private interface Probe { void run() throws IOException; }
+
+    private Capabilities withObserved(Capabilities base) {
+        Map<String, Capabilities.Support> operations = new HashMap<>(base.operations());
+        observedOperations.forEach(name -> operations.put(name, Capabilities.Support.SUPPORTED));
+        return new Capabilities(base.service(), operations, base.limits(), base.serviceVersion(),
+            base.storageMode(), base.completeOperationInventory());
+    }
+
+    private static Capabilities parseManifest(byte[] bytes) throws ProtocolException {
+        Map<String, Object> document = Json.object(Json.parse(bytes), "capability manifest");
+        if (Json.integer(document.get("schemaVersion"), "schemaVersion") != 1 ||
+            !"lunarsky-objectstore".equals(Json.string(document.get("service"), "service")))
+            throw new ProtocolException("Unsupported capability manifest");
+        String version = Json.string(document.get("serviceVersion"), "serviceVersion");
+        String mode = Json.string(document.get("storageMode"), "storageMode");
+        Map<String, Capabilities.Support> operations = new HashMap<>();
+        for (Object value : Json.array(document.get("operations"), "operations")) {
+            String name = Json.string(value, "operation name");
+            if (operations.put(name, Capabilities.Support.SUPPORTED) != null)
+                throw new ProtocolException("Duplicate capability operation");
+        }
+        Map<String, Object> rawLimits = Json.object(document.get("limits"), "limits");
+        Map<String, Long> limits = new HashMap<>();
+        for (var entry : rawLimits.entrySet()) {
+            long value = Json.integer(entry.getValue(), "limit " + entry.getKey());
+            if (value < 0) throw new ProtocolException("Negative capability limit");
+            limits.put(entry.getKey(), value);
+        }
+        for (String required : List.of("maxObjectBytes", "maxTotalBytes", "maxParts"))
+            if (!limits.containsKey(required)) throw new ProtocolException("Missing capability limit " + required);
+        return new Capabilities(Capabilities.ServiceKind.OBJECTSTORE, operations, limits, version, mode, true);
+    }
+
+    private boolean probeHealth() throws IOException {
+        URI uri = URI.create(origin() + "/health");
+        HttpRequest request = HttpRequest.newBuilder(uri).timeout(timeout).GET().build();
+        HttpResponse<InputStream> response = execute(request);
+        try (InputStream body = response.body()) {
+            if (response.statusCode() != 200) return false;
+            String value = new String(readLimited(body, 1024), StandardCharsets.UTF_8);
+            return value.matches("(?s)\\s*\\{\\s*\"status\"\\s*:\\s*\"ok\"\\s*,\\s*\"service\"\\s*:\\s*\"lunarsky-objectstore\"\\s*}\\s*");
+        }
+    }
+
+    private Element xml(String method, String path, Map<String, String> query) throws IOException {
+        HttpResponse<InputStream> response = send(method, path, query, Map.of(),
+            HttpRequest.BodyPublishers.noBody(), EMPTY_HASH);
+        return responseXml(response, null);
+    }
+
+    private static Element responseXml(HttpResponse<InputStream> response, String expectedRoot) throws IOException {
+        try (InputStream body = response.body()) {
+            if (!successful(response)) throw failure(response);
+            Element root = Xml.parse(readLimited(body, MAX_XML)).getDocumentElement();
+            if ("Error".equals(root.getLocalName())) {
+                String code = Xml.text(root, "Code");
+                throw new ObjectStorageException(response.statusCode(), code,
+                    response.headers().firstValue("x-amz-request-id").orElse(null));
+            }
+            if (expectedRoot != null) requireRoot(root, expectedRoot);
+            return root;
+        }
+    }
+
+    private HttpResponse<InputStream> send(String method, String path, Map<String, String> query,
+                                            Map<String, String> extra, HttpRequest.BodyPublisher body,
+                                            String payloadHash) throws IOException {
+        String queryString = SigV4.query(query);
+        URI uri = URI.create(origin() + path + (queryString.isEmpty() ? "" : "?" + queryString));
+        Instant now = clock.instant();
+        Map<String, String> headers = new HashMap<>(extra);
+        headers.put("host", uri.getRawAuthority().toLowerCase(Locale.ROOT));
+        headers.put("x-amz-date", SigV4.timestamp(now));
+        headers.put("x-amz-content-sha256", payloadHash);
+        String authorization = SigV4.authorization(method, uri, headers, payloadHash, now,
+            region, accessKey, secretKey);
+        HttpRequest.Builder request = HttpRequest.newBuilder(uri).timeout(timeout);
+        headers.forEach((name, value) -> {
+            if (!"host".equals(name)) request.header(name, value);
+        });
+        request.header("Authorization", authorization);
+        return execute(request.method(method, body).build());
+    }
+
+    private HttpResponse<InputStream> execute(HttpRequest request) throws IOException {
+        try { return http.send(request, HttpResponse.BodyHandlers.ofInputStream()); }
+        catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Storage request interrupted", e);
+        } catch (IOException e) { throw new TransportException(e); }
+    }
+
+    private static boolean successful(HttpResponse<?> response) {
+        return response.statusCode() >= 200 && response.statusCode() < 300;
+    }
+
+    private static ObjectStorageException failure(HttpResponse<InputStream> response) throws IOException {
+        String code = null;
+        String requestId = response.headers().firstValue("x-amz-request-id").orElse(null);
+        try {
+            byte[] bytes = readLimited(response.body(), MAX_ERROR);
+            if (bytes.length > 0) {
+                Element root = Xml.parse(bytes).getDocumentElement();
+                if ("Error".equals(root.getLocalName())) {
+                    code = Xml.text(root, "Code");
+                    if (requestId == null) requestId = Xml.text(root, "RequestId");
+                }
+            }
+        } catch (IOException ignored) {
+        }
+        return new ObjectStorageException(response.statusCode(), code, requestId);
+    }
+
+    private static byte[] readLimited(InputStream stream, int limit) throws IOException {
+        byte[] bytes = stream.readNBytes(limit + 1);
+        if (bytes.length > limit) throw new ProtocolException("Storage response exceeds size limit");
+        return bytes;
+    }
+
+    private static long length(HttpResponse<?> response) {
+        return response.headers().firstValueAsLong("content-length").orElse(-1);
+    }
+
+    private static void requireRoot(Element root, String name) throws IOException {
+        if (!name.equals(root.getLocalName())) throw new ProtocolException("Unexpected storage XML response");
+    }
+
+    private String origin() {
+        return endpoint.getScheme() + "://" + endpoint.getRawAuthority();
+    }
+
+    private static String bucketPath(String bucket) {
+        validateBucket(bucket);
+        return "/" + SigV4.encode(bucket, false);
+    }
+
+    private static String objectPath(String bucket, String key) {
+        if (key == null || key.isEmpty() || key.indexOf('\0') >= 0 ||
+            key.getBytes(StandardCharsets.UTF_8).length > 1024)
+            throw new IllegalArgumentException("object key must be 1..1024 UTF-8 bytes without NUL");
+        return bucketPath(bucket) + "/" + SigV4.encode(key, true);
+    }
+
+    private static void validateBucket(String bucket) {
+        if (bucket == null || !bucket.matches("[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]"))
+            throw new IllegalArgumentException("bucket must be a 3..63 character DNS-style name");
+    }
+
+    /** Closes the underlying HTTP client. Do not use this instance afterward. */
+    @Override public void close() { http.close(); }
+
+    /** An open response stream and selected object headers. Close this result after reading. */
+    public record ObjectData(InputStream body, long length, String contentType, String etag)
+        implements AutoCloseable {
+        @Override public void close() throws IOException { body.close(); }
+    }
+
+    /** Headers returned by a HEAD request. A negative length means it was not supplied. */
+    public record ObjectMetadata(long length, String contentType, String etag) { }
+
+    /** A listed object. */
+    public record ObjectEntry(String key, long size, String etag) { }
+
+    /** One listing page. A null next token means no next page was reported. */
+    public record ObjectPage(List<ObjectEntry> objects, String nextContinuationToken) {
+        public ObjectPage { objects = List.copyOf(objects); }
+    }
+
+    /** An unfinished multipart upload. Persist all three fields if resuming in a later process. */
+    public record MultipartUpload(String bucket, String key, String uploadId) {
+        public MultipartUpload {
+            validateBucket(bucket);
+            if (key == null || key.isEmpty()) throw new IllegalArgumentException("key is required");
+            if (uploadId == null || uploadId.isBlank()) throw new IllegalArgumentException("upload ID is required");
+        }
+    }
+
+    /** A successfully uploaded part, including the service-provided ETag. */
+    public record Part(int number, String etag, long size) {
+        public Part {
+            checkPartNumber(number);
+            if (etag == null || etag.isBlank()) throw new IllegalArgumentException("ETag is required");
+            if (size < 0) throw new IllegalArgumentException("part size must not be negative");
+        }
+    }
+
+    /** A page of uploaded parts. Pass nextPartNumberMarker to the next call when truncated. */
+    public record PartPage(List<Part> parts, boolean truncated, int nextPartNumberMarker) {
+        public PartPage { parts = List.copyOf(parts); }
+    }
+
+    /** A page of unfinished uploads. Pass both next markers to the next call when truncated. */
+    public record MultipartUploadPage(List<MultipartUpload> uploads, boolean truncated,
+                                      String nextKeyMarker, String nextUploadIdMarker) {
+        public MultipartUploadPage { uploads = List.copyOf(uploads); }
+    }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClientBuilder.java

@@ -0,0 +1,70 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.net.URI;
+import java.time.Clock;
+import java.time.Duration;
+import java.util.Objects;
+
+/** Configures a path-style, Signature V4 S3 client without third-party dependencies. */
+public final class ObjectStorageClientBuilder {
+    private URI endpoint;
+    private String region = "us-east-1";
+    private String accessKey;
+    private String secretKey;
+    private Duration timeout = Duration.ofSeconds(30);
+    private boolean allowInsecureHttp;
+
+    /** Sets the service root, for example {@code https://storage.example.com}. */
+    public ObjectStorageClientBuilder endpoint(URI value) {
+        endpoint = Objects.requireNonNull(value, "endpoint");
+        return this;
+    }
+
+    /** Sets the Signature V4 region. The default is {@code us-east-1}. */
+    public ObjectStorageClientBuilder region(String value) {
+        region = requireText(value, "region");
+        return this;
+    }
+
+    /** Sets credentials used only in request signatures. Keep them out of logs. */
+    public ObjectStorageClientBuilder credentials(String access, String secret) {
+        accessKey = requireText(access, "access key");
+        secretKey = requireText(secret, "secret key");
+        return this;
+    }
+
+    /** Sets the per-request timeout. */
+    public ObjectStorageClientBuilder timeout(Duration value) {
+        if (Objects.requireNonNull(value, "timeout").isNegative() || value.isZero())
+            throw new IllegalArgumentException("timeout must be positive");
+        timeout = value;
+        return this;
+    }
+
+    /** Allows plain HTTP for a trusted local test endpoint. HTTPS is required by default. */
+    public ObjectStorageClientBuilder allowInsecureHttp() {
+        allowInsecureHttp = true;
+        return this;
+    }
+
+    /** Builds an independent client. */
+    public ObjectStorageClient build() {
+        if (endpoint == null) throw new IllegalStateException("endpoint is required");
+        if (accessKey == null || secretKey == null) throw new IllegalStateException("credentials are required");
+        if (!region.matches("[a-z0-9-]+")) throw new IllegalArgumentException("invalid region");
+        if (!accessKey.matches("[A-Za-z0-9_+=./@-]+")) throw new IllegalArgumentException("invalid access key");
+        String scheme = endpoint.getScheme();
+        if (!"https".equalsIgnoreCase(scheme) && !(allowInsecureHttp && "http".equalsIgnoreCase(scheme)))
+            throw new IllegalArgumentException("HTTPS endpoint required unless insecure HTTP is explicitly allowed");
+        if (endpoint.getHost() == null || endpoint.getUserInfo() != null || endpoint.getRawQuery() != null ||
+            endpoint.getRawFragment() != null || !(endpoint.getRawPath() == null || endpoint.getRawPath().isEmpty() ||
+            "/".equals(endpoint.getRawPath())))
+            throw new IllegalArgumentException("endpoint must be an origin without path, query, fragment, or user info");
+        return new ObjectStorageClient(endpoint, region, accessKey, secretKey, timeout, Clock.systemUTC());
+    }
+
+    private static String requireText(String value, String name) {
+        if (value == null || value.isBlank()) throw new IllegalArgumentException(name + " is required");
+        return value;
+    }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageException.java

@@ -0,0 +1,30 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.io.IOException;
+
+/** An S3 error response with stable fields for callers to inspect. */
+public final class ObjectStorageException extends IOException {
+    private final int statusCode;
+    private final String errorCode;
+    private final String requestId;
+
+    ObjectStorageException(int statusCode, String errorCode, String requestId) {
+        super("Storage request failed: HTTP " + statusCode + (errorCode == null ? "" : " (" + errorCode + ")"));
+        this.statusCode = statusCode;
+        this.errorCode = errorCode;
+        this.requestId = requestId;
+    }
+
+    /** Returns the HTTP status. */
+    public int statusCode() { return statusCode; }
+
+    /** Returns the S3 error code, or {@code null} if the server supplied none. */
+    public String errorCode() { return errorCode; }
+
+    /** Returns the request ID, or {@code null}. */
+    public String requestId() { return requestId; }
+
+    /** Returns whether retry might help. A failed write may already have reached the server. */
+    public boolean retryable() { return statusCode == 429 || statusCode == 500 || statusCode == 502 ||
+        statusCode == 503 || statusCode == 504; }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/ProtocolException.java

@@ -0,0 +1,9 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.io.IOException;
+
+/** A successful HTTP response that does not match the expected storage protocol. */
+public final class ProtocolException extends IOException {
+    ProtocolException(String message) { super(message); }
+    ProtocolException(String message, Throwable cause) { super(message, cause); }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/SigV4.java

@@ -0,0 +1,83 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TreeMap;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+final class SigV4 {
+    private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.ROOT)
+        .withZone(ZoneOffset.UTC);
+    private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'", Locale.ROOT)
+        .withZone(ZoneOffset.UTC);
+
+    private SigV4() { }
+
+    static String hash(byte[] bytes) {
+        try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); }
+        catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
+    }
+
+    static String encode(String text, boolean keepSlash) {
+        StringBuilder result = new StringBuilder();
+        for (byte value : text.getBytes(StandardCharsets.UTF_8)) {
+            int b = value & 0xff;
+            if (b >= 'A' && b <= 'Z' || b >= 'a' && b <= 'z' || b >= '0' && b <= '9' ||
+                b == '-' || b == '_' || b == '.' || b == '~' || keepSlash && b == '/') result.append((char) b);
+            else result.append('%').append(Character.toUpperCase(Character.forDigit(b >>> 4, 16)))
+                .append(Character.toUpperCase(Character.forDigit(b & 15, 16)));
+        }
+        return result.toString();
+    }
+
+    static String query(Map<String, String> values) {
+        List<String> entries = new ArrayList<>();
+        values.forEach((key, value) -> entries.add(encode(key, false) + "=" + encode(value, false)));
+        entries.sort(String::compareTo);
+        return String.join("&", entries);
+    }
+
+    static String authorization(String method, URI uri, Map<String, String> headers, String payloadHash,
+                                Instant now, String region, String accessKey, String secretKey) {
+        TreeMap<String, String> signed = new TreeMap<>();
+        headers.forEach((name, value) -> signed.put(name.toLowerCase(Locale.ROOT),
+            value.trim().replaceAll("\\s+", " ")));
+        String names = String.join(";", signed.keySet());
+        StringBuilder canonicalHeaders = new StringBuilder();
+        signed.forEach((name, value) -> canonicalHeaders.append(name).append(':').append(value).append('\n'));
+        String canonical = method + '\n' + uri.getRawPath() + '\n' +
+            (uri.getRawQuery() == null ? "" : uri.getRawQuery()) + '\n' + canonicalHeaders + '\n' +
+            names + '\n' + payloadHash;
+        String day = DATE.format(now);
+        String scope = day + '/' + region + "/s3/aws4_request";
+        String toSign = "AWS4-HMAC-SHA256\n" + TIME.format(now) + '\n' + scope + '\n' +
+            hash(canonical.getBytes(StandardCharsets.UTF_8));
+        byte[] key = hmac(("AWS4" + secretKey).getBytes(StandardCharsets.UTF_8), day);
+        key = hmac(key, region);
+        key = hmac(key, "s3");
+        key = hmac(key, "aws4_request");
+        return "AWS4-HMAC-SHA256 Credential=" + accessKey + '/' + scope + ",SignedHeaders=" + names +
+            ",Signature=" + HexFormat.of().formatHex(hmac(key, toSign));
+    }
+
+    private static byte[] hmac(byte[] key, String value) {
+        try {
+            Mac mac = Mac.getInstance("HmacSHA256");
+            mac.init(new SecretKeySpec(key, "HmacSHA256"));
+            return mac.doFinal(value.getBytes(StandardCharsets.UTF_8));
+        } catch (Exception e) { throw new IllegalStateException("HMAC-SHA256 unavailable", e); }
+    }
+
+    static String timestamp(Instant now) { return TIME.format(now); }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/TransportException.java

@@ -0,0 +1,8 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.io.IOException;
+
+/** A connection or I/O failure. A write may have completed before this was observed. */
+public final class TransportException extends IOException {
+    TransportException(IOException cause) { super("Storage transport failed", cause); }
+}

client/src/main/java/cloud/lunarsky/objectstore/client/Xml.java

@@ -0,0 +1,54 @@
+package cloud.lunarsky.objectstore.client;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+final class Xml {
+    private Xml() { }
+
+    static Document parse(byte[] bytes) throws IOException {
+        try {
+            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+            factory.setNamespaceAware(true);
+            factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+            factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+            factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+            factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+            factory.setXIncludeAware(false);
+            factory.setExpandEntityReferences(false);
+            factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+            factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+            return factory.newDocumentBuilder().parse(new ByteArrayInputStream(bytes));
+        } catch (ParserConfigurationException | SAXException | IllegalArgumentException e) {
+            throw new ProtocolException("Malformed or unsafe XML response", e);
+        }
+    }
+
+    static Element child(Element element, String name) {
+        for (Node node = element.getFirstChild(); node != null; node = node.getNextSibling())
+            if (node instanceof Element found && name.equals(found.getLocalName())) return found;
+        return null;
+    }
+
+    static String text(Element element, String name) {
+        Element found = child(element, name);
+        return found == null ? null : found.getTextContent();
+    }
+
+    static NodeList descendants(Element element, String name) {
+        return element.getElementsByTagNameNS("*", name);
+    }
+
+    static String escape(String text) {
+        return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+            .replace("\"", "&quot;").replace("'", "&apos;");
+    }
+}

client/src/test/java/cloud/lunarsky/objectstore/client/ClientTest.java

@@ -0,0 +1,190 @@
+package cloud.lunarsky.objectstore.client;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public final class ClientTest {
+    private static final String S3_NS = "http://s3.amazonaws.com/doc/2006-03-01/";
+
+    public static void main(String[] args) throws Exception {
+        signingVector();
+        protocol();
+        System.out.println("Client tests passed");
+    }
+
+    private static void signingVector() {
+        URI uri = URI.create("https://examplebucket.s3.amazonaws.com/test.txt");
+        String empty = SigV4.hash(new byte[0]);
+        String actual = SigV4.authorization("GET", uri, Map.of(
+            "host", "examplebucket.s3.amazonaws.com", "range", "bytes=0-9",
+            "x-amz-content-sha256", empty, "x-amz-date", "20130524T000000Z"), empty,
+            Instant.parse("2013-05-24T00:00:00Z"), "us-east-1", "AKIAIOSFODNN7EXAMPLE",
+            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+        check(actual.endsWith("Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"),
+            "AWS Signature V4 vector");
+        check("a=%20&z=%2F".equals(SigV4.query(Map.of("z", "/", "a", " "))), "query encoding");
+    }
+
+    private static void protocol() throws Exception {
+        AtomicReference<String> observedHash = new AtomicReference<>();
+        AtomicReference<String> observedAcl = new AtomicReference<>();
+        AtomicReference<String> manifest = new AtomicReference<>();
+        AtomicInteger manifestStatus = new AtomicInteger(200);
+        AtomicBoolean health = new AtomicBoolean(true);
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/", exchange -> {
+            try { handle(exchange, observedHash, observedAcl, manifest, manifestStatus, health); }
+            finally { exchange.close(); }
+        });
+        server.start();
+        try (ObjectStorageClient client = new ObjectStorageClientBuilder()
+            .endpoint(URI.create("http://127.0.0.1:" + server.getAddress().getPort()))
+            .allowInsecureHttp().credentials("test-key", "test-secret").build()) {
+            client.putObject("mybucket", "hello world.txt", "hello".getBytes(StandardCharsets.UTF_8), "text/plain");
+            check(SigV4.hash("hello".getBytes(StandardCharsets.UTF_8)).equals(observedHash.get()), "payload hash");
+            try (ObjectStorageClient.ObjectData data = client.getObject("mybucket", "hello world.txt")) {
+                check("hello".equals(new String(data.body().readAllBytes(), StandardCharsets.UTF_8)), "GET body");
+            }
+            check("text/plain".equals(client.headObject("mybucket", "hello world.txt").contentType()),
+                "HEAD content type");
+            var page = client.listObjects("mybucket", "hello", null, 10);
+            check(page.objects().size() == 1 && "hello world.txt".equals(page.objects().getFirst().key()),
+                "list object key");
+            check("next-token".equals(page.nextContinuationToken()), "list continuation");
+            AclPolicy policy = client.getBucketAcl("mybucket");
+            check(policy.grants().size() == 1 && policy.grants().getFirst().permission() ==
+                AclPolicy.Permission.FULL_CONTROL, "GET ACL");
+            client.putObjectAcl("mybucket", "hello world.txt", new AclPolicy("owner-id", List.of(
+                new AclPolicy.Grant(AclPolicy.GranteeType.GROUP,
+                    "http://acs.amazonaws.com/groups/global/AllUsers", AclPolicy.Permission.READ))));
+            check(observedAcl.get().contains("<Permission>READ</Permission>"), "PUT ACL");
+            check(client.getCapabilities().service() == Capabilities.ServiceKind.OBJECTSTORE, "service discovery");
+            check(client.getCapabilities().support("versioning") == Capabilities.Support.UNKNOWN,
+                "unknown support remains unknown");
+            manifest.set("{\"schemaVersion\":1,\"service\":\"lunarsky-objectstore\"," +
+                "\"serviceVersion\":\"0.0.5\",\"storageMode\":\"disk\"," +
+                "\"operations\":[\"PutObject\",\"ListParts\"]," +
+                "\"limits\":{\"maxObjectBytes\":1024,\"maxTotalBytes\":4096,\"maxParts\":10000}}");
+            var capabilities = client.getCapabilities();
+            check(capabilities.service() == Capabilities.ServiceKind.OBJECTSTORE &&
+                capabilities.support("ListParts") == Capabilities.Support.SUPPORTED &&
+                capabilities.support("PutBucketAcl") == Capabilities.Support.UNSUPPORTED &&
+                capabilities.limits().get("maxParts") == 10000 &&
+                "0.0.5".equals(capabilities.serviceVersion()), "signed manifest");
+            manifest.set("{\"schemaVersion\":1,\"schemaVersion\":1}");
+            try {
+                client.getCapabilities();
+                throw new AssertionError("Expected malformed manifest");
+            } catch (ProtocolException expected) { }
+            health.set(false);
+            check(client.getCapabilities().service() == Capabilities.ServiceKind.UNKNOWN_S3,
+                "generic S3 with unrelated matching path");
+            manifestStatus.set(503);
+            check(client.getCapabilities().support("CopyObject") == Capabilities.Support.UNKNOWN,
+                "ambiguous manifest failure stays unknown");
+            manifestStatus.set(200);
+            manifest.set(null);
+            check(client.getCapabilities().service() == Capabilities.ServiceKind.UNKNOWN_S3, "generic S3 fallback");
+            check(client.getCapabilities().support("PutObject") == Capabilities.Support.SUPPORTED &&
+                client.getCapabilities().support("CopyObject") == Capabilities.Support.UNKNOWN,
+                "generic S3 observed support");
+            capabilities = client.probeReadOnlyCapabilities("mybucket", "hello world.txt");
+            check(capabilities.support("ListMultipartUploads") == Capabilities.Support.UNKNOWN &&
+                capabilities.support("ListObjectsV2") == Capabilities.Support.SUPPORTED,
+                "denied generic probe remains unknown");
+            client.deleteObject("mybucket", "hello world.txt");
+            try {
+                client.getObject("mybucket", "denied");
+                throw new AssertionError("Expected AccessDenied");
+            } catch (ObjectStorageException error) {
+                check(error.statusCode() == 403 && "AccessDenied".equals(error.errorCode()) &&
+                    "request-123".equals(error.requestId()), "typed S3 error");
+            }
+        } finally { server.stop(0); }
+    }
+
+    private static void handle(HttpExchange exchange, AtomicReference<String> hash,
+                               AtomicReference<String> acl, AtomicReference<String> manifest,
+                               AtomicInteger manifestStatus, AtomicBoolean health) throws IOException {
+        String path = exchange.getRequestURI().getRawPath();
+        String query = exchange.getRequestURI().getRawQuery();
+        if ("/health".equals(path)) {
+            respond(exchange, health.get() ? 200 : 404,
+                health.get() ? "{\"status\":\"ok\",\"service\":\"lunarsky-objectstore\"}" : "");
+            return;
+        }
+        check(exchange.getRequestHeaders().getFirst("Authorization") != null, "signed request");
+        if ("/_objectstore/capabilities".equals(path)) {
+            String value = manifest.get();
+            respond(exchange, value == null ? 404 : manifestStatus.get(), value == null ? "" : value);
+            return;
+        }
+        if ("/mybucket".equals(path) && query != null && query.contains("uploads=")) {
+            respond(exchange, 403, "<Error><Code>AccessDenied</Code></Error>");
+            return;
+        }
+        if ("/mybucket".equals(path) && query != null && query.contains("list-type=2")) {
+            respond(exchange, 200, "<ListBucketResult xmlns=\"" + S3_NS + "\"><Contents><Key>hello%20world.txt</Key>" +
+                "<Size>5</Size><ETag>\"abc\"</ETag></Contents><NextContinuationToken>next-token" +
+                "</NextContinuationToken></ListBucketResult>");
+            return;
+        }
+        if (query != null && query.equals("acl=")) {
+            if ("GET".equals(exchange.getRequestMethod())) {
+                respond(exchange, 200, "<AccessControlPolicy xmlns=\"" + S3_NS + "\" " +
+                    "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><Owner><ID>owner-id</ID></Owner>" +
+                    "<AccessControlList><Grant><Grantee xsi:type=\"CanonicalUser\"><ID>owner-id</ID>" +
+                    "</Grantee><Permission>FULL_CONTROL</Permission></Grant></AccessControlList></AccessControlPolicy>");
+            } else {
+                acl.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
+                check(exchange.getRequestHeaders().getFirst("Content-MD5") != null, "ACL Content-MD5");
+                respond(exchange, 200, "");
+            }
+            return;
+        }
+        if ("/mybucket/denied".equals(path)) {
+            exchange.getResponseHeaders().add("x-amz-request-id", "request-123");
+            respond(exchange, 403, "<Error><Code>AccessDenied</Code></Error>");
+            return;
+        }
+        check("/mybucket/hello%20world.txt".equals(path), "path-style key encoding");
+        switch (exchange.getRequestMethod()) {
+            case "PUT" -> {
+                byte[] body = exchange.getRequestBody().readAllBytes();
+                hash.set(exchange.getRequestHeaders().getFirst("x-amz-content-sha256"));
+                check(SigV4.hash(body).equals(hash.get()), "uploaded body hash");
+                respond(exchange, 200, "");
+            }
+            case "GET" -> respond(exchange, 200, "hello");
+            case "HEAD" -> {
+                exchange.getResponseHeaders().add("Content-Type", "text/plain");
+                exchange.sendResponseHeaders(200, -1);
+            }
+            case "DELETE" -> respond(exchange, 204, "");
+            default -> throw new AssertionError("Unexpected method");
+        }
+    }
+
+    private static void respond(HttpExchange exchange, int status, String content) throws IOException {
+        byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
+        if (status == 204 || bytes.length == 0) exchange.sendResponseHeaders(status, -1);
+        else {
+            exchange.sendResponseHeaders(status, bytes.length);
+            exchange.getResponseBody().write(bytes);
+        }
+    }
+
+    private static void check(boolean condition, String description) {
+        if (!condition) throw new AssertionError(description);
+    }
+}

client/src/test/java/cloud/lunarsky/objectstore/client/MultipartClientTest.java

@@ -0,0 +1,119 @@
+package cloud.lunarsky.objectstore.client;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+public final class MultipartClientTest {
+    public static void main(String[] args) throws Exception {
+        AtomicReference<String> completion = new AtomicReference<>();
+        AtomicBoolean completionError = new AtomicBoolean();
+        List<Integer> sizes = new ArrayList<>();
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/", exchange -> {
+            try { handle(exchange, completion, completionError, sizes); }
+            finally { exchange.close(); }
+        });
+        server.start();
+        try (ObjectStorageClient client = new ObjectStorageClientBuilder()
+            .endpoint(URI.create("http://127.0.0.1:" + server.getAddress().getPort()))
+            .allowInsecureHttp().credentials("test-key", "test-secret").build()) {
+            var upload = client.createMultipartUpload("mybucket", "large file.bin", "application/octet-stream");
+            check("u-1".equals(upload.uploadId()), "initiation ID");
+            var first = client.uploadPart(upload, 1, "first".getBytes(StandardCharsets.UTF_8));
+            var second = client.uploadPart(upload, 2, "second".getBytes(StandardCharsets.UTF_8));
+            check(first.number() == 1 && "\"etag-1\"".equals(first.etag()), "part result");
+            var page = client.listParts(upload, 0, 1);
+            check(page.truncated() && page.nextPartNumberMarker() == 1 && page.parts().size() == 1,
+                "first part page");
+            page = client.listParts(upload, page.nextPartNumberMarker(), 1);
+            check(!page.truncated() && page.parts().getFirst().number() == 2, "second part page");
+            var uploads = client.listMultipartUploads("mybucket", "large", null, null, 10);
+            check(uploads.uploads().size() == 1 && "u-1".equals(uploads.uploads().getFirst().uploadId()),
+                "unfinished upload listing");
+            String etag = client.completeMultipartUpload(upload, List.of(second, first));
+            check("\"final-etag\"".equals(etag), "completion ETag");
+            check(completion.get().indexOf("<PartNumber>1</PartNumber>") <
+                completion.get().indexOf("<PartNumber>2</PartNumber>"), "completion part order");
+            completionError.set(true);
+            try {
+                client.completeMultipartUpload(upload, List.of(first));
+                throw new AssertionError("Expected embedded S3 error");
+            } catch (ObjectStorageException error) {
+                check("InvalidPart".equals(error.errorCode()), "HTTP 200 completion error");
+            }
+            completionError.set(false);
+            Path file = Files.createTempFile("objectstore-client-multipart", ".bin");
+            try {
+                byte[] content = new byte[5 * 1024 * 1024 + 3];
+                content[content.length - 1] = 42;
+                Files.write(file, content);
+                var fileParts = client.uploadFileParts(upload, file, 5 * 1024 * 1024);
+                check(fileParts.size() == 2 && fileParts.getLast().size() == 3, "file part slicing");
+                check(sizes.contains(5 * 1024 * 1024) && sizes.contains(3), "part request sizes");
+            } finally { Files.deleteIfExists(file); }
+            client.abortMultipartUpload(upload);
+            System.out.println("Multipart client tests passed");
+        } finally { server.stop(0); }
+    }
+
+    private static void handle(HttpExchange exchange, AtomicReference<String> completion,
+                               AtomicBoolean completionError, List<Integer> sizes) throws IOException {
+        check(exchange.getRequestHeaders().getFirst("Authorization") != null, "signed multipart request");
+        String path = exchange.getRequestURI().getRawPath();
+        String query = exchange.getRequestURI().getRawQuery();
+        String method = exchange.getRequestMethod();
+        if ("/mybucket".equals(path) && "GET".equals(method) && query.contains("uploads=")) {
+            reply(exchange, 200, "<ListMultipartUploadsResult><Upload><Key>large file.bin</Key>" +
+                "<UploadId>u-1</UploadId></Upload><IsTruncated>false</IsTruncated></ListMultipartUploadsResult>");
+            return;
+        }
+        check("/mybucket/large%20file.bin".equals(path), "multipart path");
+        if ("POST".equals(method) && "uploads=".equals(query)) {
+            reply(exchange, 200, "<InitiateMultipartUploadResult><UploadId>u-1" +
+                "</UploadId></InitiateMultipartUploadResult>");
+        } else if ("PUT".equals(method) && query.contains("partNumber=")) {
+            int number = query.contains("partNumber=1") ? 1 : 2;
+            byte[] body = exchange.getRequestBody().readAllBytes();
+            check(SigV4.hash(body).equals(exchange.getRequestHeaders().getFirst("x-amz-content-sha256")),
+                "part hash");
+            sizes.add(body.length);
+            exchange.getResponseHeaders().set("ETag", "\"etag-" + number + "\"");
+            reply(exchange, 200, "");
+        } else if ("GET".equals(method) && query.contains("uploadId=")) {
+            boolean first = query.contains("part-number-marker=0");
+            reply(exchange, 200, "<ListPartsResult><Part><PartNumber>" + (first ? 1 : 2) +
+                "</PartNumber><ETag>\"etag-" + (first ? 1 : 2) + "\"</ETag><Size>" +
+                (first ? 5 : 6) + "</Size></Part><IsTruncated>" + first + "</IsTruncated>" +
+                "<NextPartNumberMarker>" + (first ? 1 : 2) + "</NextPartNumberMarker></ListPartsResult>");
+        } else if ("POST".equals(method) && query.contains("uploadId=")) {
+            completion.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
+            reply(exchange, 200, completionError.get() ? "<Error><Code>InvalidPart</Code></Error>" :
+                "<CompleteMultipartUploadResult><ETag>\"final-etag\"</ETag></CompleteMultipartUploadResult>");
+        } else if ("DELETE".equals(method) && query.contains("uploadId=")) {
+            reply(exchange, 204, "");
+        } else throw new AssertionError("Unexpected multipart operation: " + method + " " + query);
+    }
+
+    private static void reply(HttpExchange exchange, int status, String content) throws IOException {
+        byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
+        if (bytes.length == 0) exchange.sendResponseHeaders(status, -1);
+        else {
+            exchange.sendResponseHeaders(status, bytes.length);
+            exchange.getResponseBody().write(bytes);
+        }
+    }
+
+    private static void check(boolean condition, String description) {
+        if (!condition) throw new AssertionError(description);
+    }
+}

compose.cluster.encrypted.yaml

@@ -0,0 +1,114 @@
+x-encrypted-id: &encrypted-id ${ENCRYPTED_VOLUME_ID:?Set ENCRYPTED_VOLUME_ID}
+x-encrypted-data: &encrypted-data
+  ENCRYPTED_VOLUME_ID: *encrypted-id
+  ENCRYPTED_VOLUME_MARKER_FILE: /data/.objectstore-encrypted
+x-encrypted-staging: &encrypted-staging
+  ENCRYPTED_VOLUME_ID: *encrypted-id
+  ENCRYPTED_VOLUME_MARKER_FILE: /tmp/.objectstore-encrypted
+x-metadata-entrypoint: &metadata-entrypoint
+  - /bin/sh
+  - -ec
+  - |
+    test -f /var/lib/postgresql/data/.objectstore-encrypted &&
+    test ! -L /var/lib/postgresql/data/.objectstore-encrypted &&
+    printf '%s\n' "$(printenv ENCRYPTED_VOLUME_ID)" | grep -Eq '^[a-f0-9]{32}$' &&
+    grep -Fxq -e "$(printenv ENCRYPTED_VOLUME_ID)" /var/lib/postgresql/data/.objectstore-encrypted ||
+      { echo 'Encrypted metadata volume unavailable' >&2; exit 1; }
+    exec docker-entrypoint.sh postgres
+
+services:
+  gateway:
+    environment: *encrypted-staging
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/gateway-staging
+        target: /tmp
+        bind:
+          create_host_path: false
+
+  metadata:
+    environment:
+      ENCRYPTED_VOLUME_ID: *encrypted-id
+      PGDATA: /var/lib/postgresql/data/pgdata
+    entrypoint: *metadata-entrypoint
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/metadata
+        target: /var/lib/postgresql/data
+        bind:
+          create_host_path: false
+
+  metadata-recovery:
+    environment:
+      ENCRYPTED_VOLUME_ID: *encrypted-id
+      PGDATA: /var/lib/postgresql/data/pgdata
+    entrypoint: *metadata-entrypoint
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/metadata-recovery
+        target: /var/lib/postgresql/data
+        bind:
+          create_host_path: false
+
+  repair:
+    environment: *encrypted-staging
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/repair-staging
+        target: /tmp
+        bind:
+          create_host_path: false
+
+  gc:
+    environment: *encrypted-staging
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/gc-staging
+        target: /tmp
+        bind:
+          create_host_path: false
+
+  maintenance:
+    environment: *encrypted-staging
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/maintenance-staging
+        target: /tmp
+        bind:
+          create_host_path: false
+
+  node-a:
+    environment: *encrypted-data
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/node-a
+        target: /data
+        bind:
+          create_host_path: false
+
+  node-b:
+    environment: *encrypted-data
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/node-b
+        target: /data
+        bind:
+          create_host_path: false
+
+  node-c:
+    environment: *encrypted-data
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/node-c
+        target: /data
+        bind:
+          create_host_path: false
+
+  node-d:
+    environment: *encrypted-data
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/cluster/node-d
+        target: /data
+        bind:
+          create_host_path: false

compose.cluster.tls.yaml

@@ -0,0 +1,58 @@
+x-client-tls: &client-tls
+  CLUSTER_NODES: https://node-a:9100,https://node-b:9100,https://node-c:9100
+  CLUSTER_TLS_TRUSTSTORE: /run/objectstore-tls/trust.p12
+  CLUSTER_TLS_PASSWORD_FILE: /run/objectstore-tls/trust.pass
+
+x-tls-volume: &tls-volume
+  - ${CLUSTER_TLS_DIR:?Set CLUSTER_TLS_DIR to a private certificate directory}:/run/objectstore-tls:ro
+
+x-node-health: &node-health
+  test: ["CMD", "nc", "-z", "-w", "2", "127.0.0.1", "9100"]
+  interval: 10s
+  timeout: 3s
+  retries: 3
+
+services:
+  gateway:
+    environment: *client-tls
+    volumes: *tls-volume
+
+  repair:
+    environment: *client-tls
+    volumes: *tls-volume
+
+  gc:
+    environment: *client-tls
+    volumes: *tls-volume
+
+  maintenance:
+    environment: *client-tls
+    volumes: *tls-volume
+
+  node-a:
+    environment:
+      NODE_TLS_KEYSTORE: /run/objectstore-tls/node-a.p12
+      NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-a.pass
+    volumes: *tls-volume
+    healthcheck: *node-health
+
+  node-b:
+    environment:
+      NODE_TLS_KEYSTORE: /run/objectstore-tls/node-b.p12
+      NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-b.pass
+    volumes: *tls-volume
+    healthcheck: *node-health
+
+  node-c:
+    environment:
+      NODE_TLS_KEYSTORE: /run/objectstore-tls/node-c.p12
+      NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-c.pass
+    volumes: *tls-volume
+    healthcheck: *node-health
+
+  node-d:
+    environment:
+      NODE_TLS_KEYSTORE: /run/objectstore-tls/node-d.p12
+      NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-d.pass
+    volumes: *tls-volume
+    healthcheck: *node-health

compose.cluster.yaml

@@ -12,11 +12,21 @@ services:
       S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
       S3_BUCKET: ${S3_BUCKET:-objects}
       S3_REGION: ${S3_REGION:-us-east-1}
+      S3_CREDENTIALS_FILE: ${S3_CREDENTIALS_FILE:-}
       MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728}
       MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648}
+      MAX_IN_FLIGHT_REQUESTS: ${MAX_IN_FLIGHT_REQUESTS:-16}
+      HTTP_BACKLOG: ${HTTP_BACKLOG:-64}
+      S3_VIRTUAL_HOST_SUFFIX: ${S3_VIRTUAL_HOST_SUFFIX:-}
+      PUBLIC_REQUESTS_PER_SECOND: ${PUBLIC_REQUESTS_PER_SECOND:-0}
+      PUBLIC_REQUEST_BURST: ${PUBLIC_REQUEST_BURST:-}
+      PUBLIC_BYTES_PER_SECOND: ${PUBLIC_BYTES_PER_SECOND:-0}
+      PUBLIC_BYTE_BURST: ${PUBLIC_BYTE_BURST:-}
+      PUBLIC_MAX_IN_FLIGHT_PER_IP: ${PUBLIC_MAX_IN_FLIGHT_PER_IP:-}
+      PUBLIC_TRUSTED_PROXY_IPS: ${PUBLIC_TRUSTED_PROXY_IPS:-}
       CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
       CLUSTER_NODES: ${CLUSTER_NODES:-http://node-a:9100,http://node-b:9100,http://node-c:9100}
-      POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10
+      POSTGRES_JDBC_URL: ${POSTGRES_JDBC_URL:-jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10&targetServerType=primary}
       POSTGRES_USER: objectstore
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
     ports:
@@ -31,7 +41,7 @@ services:
       node-c:
         condition: service_healthy
     healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9000/ready"]
+      test: ["CMD", "wget", "-qO-", "--header", "X-Real-IP: 127.0.0.1", "http://127.0.0.1:9000/ready"]
       interval: 10s
       timeout: 4s
       retries: 3
@@ -59,7 +69,25 @@ services:
     security_opt:
       - no-new-privileges:true
 
-  repair:
+  metadata-recovery:
+    image: postgres:17-alpine
+    profiles: [recovery]
+    environment:
+      POSTGRES_DB: objectstore
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    volumes:
+      - cluster-metadata-recovery:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U objectstore -d objectstore"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    mem_limit: 256m
+    security_opt:
+      - no-new-privileges:true
+
+  repair: &repair
     image: lunarsky-objectstore:cluster-local
     profiles: [maintenance]
     entrypoint: ["/usr/local/bin/objectstore", "cluster-repair"]
@@ -71,9 +99,57 @@ services:
       CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
       CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN}
       S3_BUCKET: ${S3_BUCKET:-objects}
-      POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10
+      POSTGRES_JDBC_URL: ${POSTGRES_JDBC_URL:-jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10&targetServerType=primary}
       POSTGRES_USER: objectstore
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+      MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728}
+      MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648}
+      CLUSTER_GC_MIN_AGE_SECONDS: ${CLUSTER_GC_MIN_AGE_SECONDS:-1209600}
+      CLUSTER_BACKUP_RETENTION_SECONDS: ${CLUSTER_BACKUP_RETENTION_SECONDS:-0}
+      CLUSTER_GC_TEST_MODE: ${CLUSTER_GC_TEST_MODE:-false}
+    mem_limit: 384m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
+  gc:
+    <<: *repair
+    entrypoint: ["/usr/local/bin/objectstore", "cluster-gc"]
+
+  maintenance:
+    image: lunarsky-objectstore:cluster-local
+    profiles: [automatic]
+    restart: unless-stopped
+    entrypoint: ["/usr/local/bin/objectstore", "cluster-repair", "--loop"]
+    environment:
+      STORE_MODE: cluster
+      CLUSTER_LOCAL_DEV: "true"
+      CLUSTER_TEST_NODE_DOMAINS: "true"
+      CLUSTER_NODES: ${CLUSTER_NODES:-http://node-a:9100,http://node-b:9100,http://node-c:9100}
+      CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
+      CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN}
+      CLUSTER_MAINTENANCE_INTERVAL_SECONDS: ${CLUSTER_MAINTENANCE_INTERVAL_SECONDS:-60}
+      CLUSTER_GC_ENABLED: ${CLUSTER_GC_ENABLED:-false}
+      CLUSTER_GC_INTERVAL_SECONDS: ${CLUSTER_GC_INTERVAL_SECONDS:-86400}
+      CLUSTER_GC_MIN_AGE_SECONDS: ${CLUSTER_GC_MIN_AGE_SECONDS:-1209600}
+      CLUSTER_BACKUP_RETENTION_SECONDS: ${CLUSTER_BACKUP_RETENTION_SECONDS:-0}
+      CLUSTER_GC_TEST_MODE: ${CLUSTER_GC_TEST_MODE:-false}
+      S3_BUCKET: ${S3_BUCKET:-objects}
+      POSTGRES_JDBC_URL: ${POSTGRES_JDBC_URL:-jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10&targetServerType=primary}
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+      MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728}
+      MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648}
+    depends_on:
+      metadata:
+        condition: service_healthy
+      node-a:
+        condition: service_healthy
+      node-b:
+        condition: service_healthy
+      node-c:
+        condition: service_healthy
     mem_limit: 384m
     security_opt:
       - no-new-privileges:true
@@ -121,6 +197,7 @@ services:
 
 volumes:
   cluster-metadata:
+  cluster-metadata-recovery:
   cluster-node-a:
   cluster-node-b:
   cluster-node-c:

compose.encrypted.yaml

@@ -0,0 +1,11 @@
+services:
+  objectstore:
+    environment:
+      ENCRYPTED_VOLUME_ID: ${ENCRYPTED_VOLUME_ID:?Set ENCRYPTED_VOLUME_ID}
+      ENCRYPTED_VOLUME_MARKER_FILE: /data/.objectstore-encrypted
+    volumes:
+      - type: bind
+        source: ${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT}/single
+        target: /data
+        bind:
+          create_host_path: false

compose.yaml

@@ -8,8 +8,18 @@ services:
       S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env}
       S3_BUCKET: ${S3_BUCKET:-objects}
       S3_REGION: ${S3_REGION:-us-east-1}
+      S3_CREDENTIALS_FILE: ${S3_CREDENTIALS_FILE:-}
       MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728}
       MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648}
+      MAX_IN_FLIGHT_REQUESTS: ${MAX_IN_FLIGHT_REQUESTS:-16}
+      HTTP_BACKLOG: ${HTTP_BACKLOG:-64}
+      S3_VIRTUAL_HOST_SUFFIX: ${S3_VIRTUAL_HOST_SUFFIX:-}
+      PUBLIC_REQUESTS_PER_SECOND: ${PUBLIC_REQUESTS_PER_SECOND:-0}
+      PUBLIC_REQUEST_BURST: ${PUBLIC_REQUEST_BURST:-}
+      PUBLIC_BYTES_PER_SECOND: ${PUBLIC_BYTES_PER_SECOND:-0}
+      PUBLIC_BYTE_BURST: ${PUBLIC_BYTE_BURST:-}
+      PUBLIC_MAX_IN_FLIGHT_PER_IP: ${PUBLIC_MAX_IN_FLIGHT_PER_IP:-}
+      PUBLIC_TRUSTED_PROXY_IPS: ${PUBLIC_TRUSTED_PROXY_IPS:-}
     ports:
       - "127.0.0.1:${HOST_PORT:-9000}:9000"
     volumes:

lib/LICENSE.hash4j

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

scripts/backup-cluster-metadata.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+env_file=${1:?Usage: sh scripts/backup-cluster-metadata.sh /path/to/cluster.env /path/to/metadata.dump}
+output=${2:?Usage: sh scripts/backup-cluster-metadata.sh /path/to/cluster.env /path/to/metadata.dump}
+if [ -e "$output" ]; then
+  echo "Backup destination already exists" >&2
+  exit 1
+fi
+umask 077
+temporary=$(mktemp "${output}.tmp.XXXXXX")
+trap 'rm -f "$temporary"' EXIT
+docker compose --env-file "$env_file" -f compose.cluster.yaml exec -T metadata \
+  pg_dump -U objectstore -d objectstore --format=custom --no-owner --no-acl > "$temporary"
+test -s "$temporary"
+docker compose --env-file "$env_file" -f compose.cluster.yaml exec -T metadata \
+  pg_restore --list < "$temporary" > /dev/null
+mv "$temporary" "$output"
+echo "Metadata backup written to $output"

scripts/objectstore

@@ -1,14 +1,18 @@
 #!/bin/sh
 if [ "${1:-}" = "cluster-repair" ]; then
   shift
-  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterRepair "$@"
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterRepair "$@"
+fi
+if [ "${1:-}" = "cluster-gc" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterGc "$@"
 fi
 if [ "${1:-}" = "cluster-migrate" ]; then
   shift
-  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterMigrate "$@"
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterMigrate "$@"
 fi
 if [ "${1:-}" = "cluster-join" ]; then
   shift
-  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterJoin "$@"
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterJoin "$@"
 fi
-exec java -XX:MaxRAMPercentage=70 -cp /app cloud.lunarsky.store.Cli "$@"
+exec java -XX:MaxRAMPercentage=70 -cp /app:/app/hash4j.jar cloud.lunarsky.store.Cli "$@"

scripts/prepare-encrypted-storage.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+set -eu
+
+mode=${1:-}
+case "$mode" in
+  single) directories='single' ;;
+  cluster) directories='cluster/gateway-staging cluster/metadata cluster/metadata-recovery cluster/repair-staging cluster/gc-staging cluster/maintenance-staging cluster/node-a cluster/node-b cluster/node-c cluster/node-d' ;;
+  *) echo 'Usage: prepare-encrypted-storage.sh single|cluster [environment-file]' >&2; exit 2 ;;
+esac
+
+if [ "$#" -gt 2 ]; then
+  echo 'Too many arguments' >&2
+  exit 2
+fi
+if [ "$#" -eq 2 ]; then
+  [ -r "$2" ] || { echo 'Environment file is not readable' >&2; exit 2; }
+  while IFS='=' read -r name value || [ -n "${name:-}" ]; do
+    case "$name" in
+      ENCRYPTED_STORAGE_ROOT) ENCRYPTED_STORAGE_ROOT=$value ;;
+      ENCRYPTED_VOLUME_ID) ENCRYPTED_VOLUME_ID=$value ;;
+    esac
+  done < "$2"
+fi
+
+root=${ENCRYPTED_STORAGE_ROOT:?Set ENCRYPTED_STORAGE_ROOT to an existing mounted LUKS directory}
+id=${ENCRYPTED_VOLUME_ID:?Set ENCRYPTED_VOLUME_ID to 32 lowercase hex characters}
+case "$id" in *[!0-9a-f]*|'') echo 'ENCRYPTED_VOLUME_ID must be lowercase hex' >&2; exit 2 ;; esac
+[ "${#id}" -eq 32 ] || { echo 'ENCRYPTED_VOLUME_ID must be 32 characters' >&2; exit 2; }
+[ -d "$root" ] && [ ! -L "$root" ] || { echo 'Encrypted root is missing or a symlink' >&2; exit 1; }
+root=$(realpath -e "$root")
+source=$(findmnt -n -M "$root" -o SOURCE) || { echo 'Encrypted root is not a mount point' >&2; exit 1; }
+device=$(printf '%s\n' "$source" | sed 's/\[.*$//')
+case "$device" in /dev/*) ;; *) echo 'Encrypted root must be backed by a block device' >&2; exit 1 ;; esac
+lsblk -s -n -r -o TYPE "$device" | grep -Fxq crypt || {
+  echo 'Encrypted root is not backed by an active dm-crypt mapping' >&2
+  exit 1
+}
+
+for relative in $directories; do
+  path=$root/$relative
+  [ ! -L "$root/cluster" ] || { echo 'Refusing symlink under encrypted root' >&2; exit 1; }
+  [ ! -L "$path" ] || { echo "Refusing symlink: $path" >&2; exit 1; }
+  if [ -d "$path" ] && [ ! -e "$path/.objectstore-encrypted" ] &&
+     [ -n "$(find "$path" -mindepth 1 -maxdepth 1 -print -quit)" ]; then
+    echo "Refusing to mark nonempty directory: $path" >&2
+    exit 1
+  fi
+  install -d -m 0700 "$path"
+  [ "$(findmnt -n -T "$path" -o TARGET)" = "$root" ] || {
+    echo "Directory is not on the encrypted mount: $path" >&2
+    exit 1
+  }
+  marker=$path/.objectstore-encrypted
+  if [ -e "$marker" ]; then
+    [ ! -L "$marker" ] && [ "$(cat "$marker")" = "$id" ] || {
+      echo "Encrypted volume marker mismatch: $path" >&2
+      exit 1
+    }
+  else
+    printf '%s\n' "$id" > "$marker"
+  fi
+  chmod 0600 "$marker"
+  case "$relative" in
+    cluster/metadata|cluster/metadata-recovery) chown 70:70 "$path" "$marker" ;;
+    *) chown 10001:10001 "$path" "$marker" ;;
+  esac
+done
+
+echo "Prepared $mode paths on the active encrypted mount: $root"

scripts/test-cluster-http.py

@@ -1,12 +1,16 @@
 #!/usr/bin/env python3
 import datetime
+import base64
 import hashlib
 import hmac
+import http.client
+import json
 import pathlib
+import re
 import sys
-import urllib.error
 import urllib.parse
-import urllib.request
+import zlib
+import xml.etree.ElementTree as ET
 
 
 values = dict(line.strip().split("=", 1) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()
@@ -14,14 +18,35 @@ values = dict(line.strip().split("=", 1) for line in pathlib.Path(sys.argv[1]).r
 access = values["S3_ACCESS_KEY"]
 secret = values["S3_SECRET_KEY"]
 bucket = values.get("S3_BUCKET", "objects")
-port = values.get("CLUSTER_HOST_PORT", "9001")
+port = int(values.get("CLUSTER_HOST_PORT", "9001"))
+if not 1 <= port <= 65535:
+    raise ValueError("CLUSTER_HOST_PORT must be between 1 and 65535")
 host = f"127.0.0.1:{port}"
+MAX_RESPONSE_BYTES = 1024 * 1024
 
 
 def sign(key, message):
     return hmac.new(key, message.encode(), hashlib.sha256).digest()
 
 
+def parse_xml(content):
+    if len(content) > MAX_RESPONSE_BYTES:
+        raise ValueError("Unsafe XML response from test server")
+    text = content.decode("utf-8")
+    if "<!DOCTYPE" in text or "<!ENTITY" in text:
+        raise ValueError("Unsafe XML response from test server")
+    parser = ET.XMLParser()
+    parser.feed(text)
+    return parser.close()
+
+
+def read_response(response):
+    content = response.read(MAX_RESPONSE_BYTES + 1)
+    if len(content) > MAX_RESPONSE_BYTES:
+        raise ValueError("Oversized response from test server")
+    return content
+
+
 def request(method, path, body=b"", extra=None):
     extra = extra or {}
     date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
@@ -40,14 +65,23 @@ def request(method, path, body=b"", extra=None):
     signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
     headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={access}/{scope},"
                                 f"SignedHeaders={signed_names},Signature={signature}")
-    url = f"http://{host}{path}"
+    connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
-    outgoing = urllib.request.Request(url, data=body if method == "PUT" else None,
-                                      method=method, headers=headers)
     try:
-        with urllib.request.urlopen(outgoing, timeout=30) as response:
+        connection.request(method, path, body=body if method in ("PUT", "POST") else None, headers=headers)
-            return response.status, response.read(), response.headers
+        response = connection.getresponse()
-    except urllib.error.HTTPError as error:
+        return response.status, read_response(response), response.headers
-        return error.code, error.read(), error.headers
+    finally:
+        connection.close()
+
+
+def anonymous(method, path):
+    connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
+    try:
+        connection.request(method, path)
+        response = connection.getresponse()
+        return response.status, read_response(response)
+    finally:
+        connection.close()
 
 
 if len(sys.argv) > 2 and sys.argv[2] == "survivor":
@@ -56,6 +90,26 @@ if len(sys.argv) > 2 and sys.argv[2] == "survivor":
     print("Cluster surviving-replica HTTP read passed")
     sys.exit(0)
 
+if len(sys.argv) > 2 and sys.argv[2] == "acl":
+    path = f"/{bucket}/cluster-test/acl-multipart"
+    status, content, _ = request("POST", path + "?uploads", extra={"x-amz-acl": "public-read"})
+    assert status == 200, (status, content)
+    upload_id = parse_xml(content).findtext("UploadId")
+    status, _, headers = request("PUT", path + f"?partNumber=1&uploadId={upload_id}", b"public part")
+    assert status == 200, status
+    completion = ("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
+                  headers["etag"] + "</ETag></Part></CompleteMultipartUpload>").encode()
+    status, _, _ = request("POST", path + f"?uploadId={upload_id}", completion)
+    assert status == 200, status
+    status, content = anonymous("GET", path)
+    assert status == 200 and content == b"public part", (status, content)
+    status, _, _ = request("PUT", path, b"private replacement")
+    assert status == 200, status
+    status, _ = anonymous("GET", path)
+    assert status == 403, status
+    print("Cluster multipart ACL and replacement tests passed")
+    sys.exit(0)
+
 if len(sys.argv) > 4 and sys.argv[2] == "status":
     key = urllib.parse.quote(sys.argv[3], safe="/")
     expected = int(sys.argv[4])
@@ -64,6 +118,35 @@ if len(sys.argv) > 4 and sys.argv[2] == "status":
     print(f"Cluster GET status passed: {status}")
     sys.exit(0)
 
+if len(sys.argv) > 2 and sys.argv[2] == "version-survivor":
+    status, listing, _ = request("GET", "/version-bucket?versions")
+    assert status == 200, status
+    root = parse_xml(listing)
+    namespace = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
+    expected_etag = '"' + hashlib.md5(b"older cluster version", usedforsecurity=False).hexdigest() + '"'
+    versions = [version for version in root.findall("s3:Version", namespace)
+                if version.findtext("s3:ETag", namespaces=namespace) == expected_etag]
+    assert len(versions) == 1, listing
+    version_id = versions[0].findtext("s3:VersionId", namespaces=namespace)
+    status, content, _ = request("GET", "/version-bucket/note.txt?versionId=" + version_id)
+    assert status == 200 and content == b"older cluster version", (status, content)
+    multipart_versions = [version for version in root.findall("s3:Version", namespace)
+                          if version.findtext("s3:Key", namespaces=namespace) == "multipart.txt"]
+    assert len(multipart_versions) == 1, listing
+    multipart_id = multipart_versions[0].findtext("s3:VersionId", namespaces=namespace)
+    status, content, _ = request("GET", "/version-bucket/multipart.txt?versionId=" + multipart_id)
+    assert status == 200 and content == b"retained multipart version", (status, content)
+    print("Cluster historical version survived repair and cleanup")
+    sys.exit(0)
+
+
+status, content, headers = request("GET", "/_objectstore/capabilities")
+capabilities = json.loads(content)
+assert status == 200 and capabilities["schemaVersion"] == 1, (status, content)
+assert capabilities["storageMode"] == "cluster", capabilities
+assert "ListParts" in capabilities["operations"], capabilities
+assert capabilities["limits"]["maxObjectBytes"] > 0, capabilities
+assert headers.get_content_type() == "application/json", headers
 
 key = f"/{bucket}/cluster-test/http.txt"
 body = b"HTTP gateway integration test"
@@ -81,4 +164,168 @@ status, _, _ = request("DELETE", key)
 assert status == 204, status
 status, _, _ = request("GET", key)
 assert status == 404, status
-print("Cluster HTTP tests passed: signed PUT, GET, range, HEAD, LIST, DELETE")
+
+public_key = f"/{bucket}/cluster-test/public.txt"
+status, _, _ = request("PUT", public_key, b"public object", {"x-amz-acl": "public-read"})
+assert status == 200, status
+status, content = anonymous("GET", public_key)
+assert status == 200 and content == b"public object", (status, content)
+status, acl, _ = request("GET", public_key + "?acl")
+assert status == 200 and b"AllUsers" in acl, (status, acl)
+status, _, _ = request("PUT", public_key, b"private replacement")
+assert status == 200, status
+status, _ = anonymous("GET", public_key)
+assert status == 403, status
+status, _, _ = request("PUT", public_key + "?acl", extra={"x-amz-acl": "public-read"})
+assert status == 200, status
+status, content = anonymous("GET", public_key)
+assert status == 200 and content == b"private replacement", (status, content)
+
+copy_source = f"/{bucket}/cluster-test/copy-source.txt"
+copy_target = f"/{bucket}/cluster-test/copied.txt"
+body = b"cluster copy and checksum test"
+crc32 = base64.b64encode(zlib.crc32(body).to_bytes(4, "big")).decode()
+md5 = base64.b64encode(hashlib.md5(body, usedforsecurity=False).digest()).decode()
+status, _, headers = request("PUT", copy_source, body,
+                             {"content-type": "text/plain", "content-md5": md5,
+                              "x-amz-checksum-crc32": crc32,
+                              "x-amz-sdk-checksum-algorithm": "CRC32"})
+assert status == 200 and headers["x-amz-checksum-crc32"] == crc32, status
+status, _, headers = request("HEAD", copy_source,
+                             extra={"x-amz-checksum-mode": "ENABLED"})
+assert status == 200 and headers["x-amz-checksum-crc32"] == crc32, status
+status, content, _ = request("PUT", copy_source, body,
+                             {"content-md5": base64.b64encode(bytes(16)).decode()})
+assert status == 400 and b"BadDigest" in content, (status, content)
+status, content, _ = request("GET", copy_source)
+assert status == 200 and content == body, (status, content)
+status, content, _ = request("PUT", copy_target, extra={"x-amz-copy-source": copy_source})
+assert status == 200 and b"<CopyObjectResult>" in content, (status, content)
+status, content, headers = request("GET", copy_target)
+assert status == 200 and content == body and headers["content-type"] == "text/plain", (status, content)
+status, _, headers = request("HEAD", copy_target, extra={"x-amz-checksum-mode": "ENABLED"})
+assert status == 200 and headers["x-amz-checksum-crc32"] == crc32, status
+status, _, _ = request("DELETE", copy_source)
+assert status == 204, status
+status, _, _ = request("DELETE", copy_target)
+assert status == 204, status
+
+attribute_key = f"/{bucket}/cluster-test/attributes.txt"
+status, _, _ = request("PUT", attribute_key, b"cluster attributes",
+                       {"x-amz-meta-project": "LunarSky", "x-amz-tagging": "stage=one"})
+assert status == 200, status
+status, content, headers = request("GET", attribute_key)
+assert status == 200 and content == b"cluster attributes", (status, content)
+assert headers["x-amz-meta-project"] == "LunarSky" and headers["x-amz-tagging-count"] == "1"
+status, content, _ = request("GET", attribute_key + "?tagging")
+assert status == 200 and b"<Key>stage</Key><Value>one</Value>" in content, (status, content)
+replacement = b"<Tagging><TagSet><Tag><Key>stage</Key><Value>two</Value></Tag></TagSet></Tagging>"
+status, _, _ = request("PUT", attribute_key + "?tagging", replacement)
+assert status == 200, status
+status, content, _ = request("GET", attribute_key + "?tagging")
+assert status == 200 and b"<Value>two</Value>" in content, (status, content)
+status, _, _ = request("DELETE", attribute_key + "?tagging")
+assert status == 204, status
+status, content, _ = request("GET", attribute_key + "?tagging")
+assert status == 200 and b"<TagSet></TagSet>" in content, (status, content)
+status, content, _ = request("GET", "/")
+assert status == 200 and f"<Name>{bucket}</Name>".encode() in content, (status, content)
+status, _, _ = request("PUT", "/second-bucket")
+assert status == 200, status
+status, _, _ = request("PUT", "/second-bucket/one.txt", b"second bucket")
+assert status == 200, status
+status, content, _ = request("PUT", "/second-bucket/copied.txt", extra={"x-amz-copy-source": attribute_key})
+assert status == 200 and b"<CopyObjectResult>" in content, (status, content)
+status, content, headers = request("GET", "/second-bucket/copied.txt")
+assert status == 200 and content == b"cluster attributes", (status, content)
+assert headers["x-amz-meta-project"] == "LunarSky"
+status, _, _ = request("DELETE", attribute_key)
+assert status == 204, status
+status, content, _ = request("GET", "/second-bucket/one.txt")
+assert status == 200 and content == b"second bucket", (status, content)
+status, _, _ = request("DELETE", "/second-bucket")
+assert status == 409, status
+status, _, _ = request("DELETE", "/second-bucket/one.txt")
+assert status == 204, status
+status, _, _ = request("DELETE", "/second-bucket/copied.txt")
+assert status == 204, status
+status, _, _ = request("DELETE", "/second-bucket")
+assert status == 204, status
+
+status, _, _ = request("PUT", "/version-bucket")
+assert status == 200, status
+version_key = "/version-bucket/note.txt"
+status, _, _ = request("PUT", version_key, b"pre-versioning")
+assert status == 200, status
+versioning = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"
+status, _, _ = request("PUT", "/version-bucket?versioning", versioning)
+assert status == 200, status
+status, _, headers = request("PUT", version_key, b"older cluster version")
+assert status == 200, status
+old_version = headers["x-amz-version-id"]
+status, _, headers = request("PUT", version_key, b"newer cluster version")
+assert status == 200 and headers["x-amz-version-id"] != old_version, status
+status, content, _ = request("GET", version_key + "?versionId=" + old_version)
+assert status == 200 and content == b"older cluster version", (status, content)
+status, _, headers = request("DELETE", version_key)
+assert status == 204 and headers["x-amz-delete-marker"] == "true", status
+status, content, _ = request("GET", version_key)
+assert status == 404, (status, content)
+status, content, _ = request("GET", "/version-bucket?versions")
+assert status == 200 and b"<DeleteMarker>" in content and old_version.encode() in content, (status, content)
+versioned_multipart = "/version-bucket/multipart.txt"
+status, content, _ = request("POST", versioned_multipart + "?uploads")
+assert status == 200, (status, content)
+versioned_upload = parse_xml(content).findtext("UploadId")
+assert versioned_upload, content
+versioned_part = b"retained multipart version"
+status, _, headers = request("PUT", versioned_multipart +
+                             f"?partNumber=1&uploadId={versioned_upload}", versioned_part)
+assert status == 200, status
+completion = ("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
+              headers["etag"] + "</ETag></Part></CompleteMultipartUpload>").encode()
+status, _, headers = request("POST", versioned_multipart + f"?uploadId={versioned_upload}", completion)
+assert status == 200 and headers.get("x-amz-version-id"), status
+versioned_part_id = headers["x-amz-version-id"]
+status, _, _ = request("DELETE", versioned_multipart)
+assert status == 204, status
+status, content, _ = request("GET", versioned_multipart + "?versionId=" + versioned_part_id)
+assert status == 200 and content == versioned_part, (status, content)
+
+multipart_key = f"/{bucket}/cluster-test/http-multipart.txt"
+status, content, _ = request("POST", multipart_key + "?uploads",
+                             extra={"x-amz-meta-project": "multipart", "x-amz-tagging": "stage=upload",
+                                    "x-amz-acl": "public-read"})
+assert status == 200, (status, content)
+match = re.search(rb"<UploadId>([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})</UploadId>",
+                  content[:8192])
+assert match, content
+upload_id = match.group(1).decode("ascii")
+part_etags = []
+for number, part in enumerate((b"hello ", b"world"), start=1):
+    checksum = base64.b64encode(hashlib.sha1(part).digest()).decode()
+    status, _, headers = request("PUT", multipart_key + f"?partNumber={number}&uploadId={upload_id}",
+                                 part, {"x-amz-checksum-sha1": checksum})
+    assert status == 200, status
+    assert headers["x-amz-checksum-sha1"] == checksum, headers
+    part_etags.append(headers["etag"])
+status, content, _ = request("GET", multipart_key + f"?uploadId={upload_id}&max-parts=1")
+assert status == 200 and b"<IsTruncated>true</IsTruncated>" in content, (status, content)
+status, content, _ = request("GET", f"/{bucket}?uploads&prefix=cluster-test%2Fhttp-multipart")
+assert status == 200 and upload_id.encode() in content, (status, content)
+completion = "<CompleteMultipartUpload>" + "".join(
+    f"<Part><PartNumber>{number}</PartNumber><ETag>{etag}</ETag></Part>"
+    for number, etag in enumerate(part_etags, start=1)) + "</CompleteMultipartUpload>"
+status, content, _ = request("POST", multipart_key + f"?uploadId={upload_id}", completion.encode(),
+                             {"content-type": "application/xml"})
+assert status == 200 and b"<CompleteMultipartUploadResult>" in content, (status, content)
+status, content, headers = request("GET", multipart_key)
+assert status == 200 and content == b"hello world", (status, content)
+status, public_content = anonymous("GET", multipart_key)
+assert status == 200 and public_content == b"hello world", (status, public_content)
+assert headers["x-amz-meta-project"] == "multipart" and headers["x-amz-tagging-count"] == "1"
+status, _, headers = request("HEAD", multipart_key, extra={"x-amz-checksum-mode": "ENABLED"})
+assert status == 200 and len(base64.b64decode(headers["x-amz-checksum-crc64nvme"])) == 8, status
+status, content, _ = request("GET", f"/{bucket}?uploads&prefix=cluster-test%2Fhttp-multipart")
+assert status == 200 and upload_id.encode() not in content, (status, content)
+print("Cluster HTTP tests passed: signed objects, copies, checksums, multipart, and versioning")

scripts/test-cluster.sh

@@ -3,13 +3,20 @@ set -eu
 cd "$(dirname "$0")/.."
 env_file=${1:?Usage: sh scripts/test-cluster.sh /path/to/local-cluster.env}
 host_port=${CLUSTER_HOST_PORT:-9001}
+backup_dir=
 compose() { docker compose --env-file "$env_file" -f compose.cluster.yaml "$@"; }
-restore() { compose start metadata node-a node-b >/dev/null 2>&1 || true; }
+restore() {
+  compose stop maintenance >/dev/null 2>&1 || true
+  compose start metadata node-a node-b node-c >/dev/null 2>&1 || true
+  compose exec -T metadata psql -U objectstore -d postgres -c \
+    'ALTER DATABASE objectstore RESET default_transaction_read_only' >/dev/null 2>&1 || true
+  if [ -n "$backup_dir" ]; then rm -rf "$backup_dir"; fi
+}
 trap restore EXIT
 compose up -d --build
 run_phase() {
   compose exec -T gateway java --add-modules jdk.httpserver,java.net.http \
-    -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterIntegrationTest "$1"
+    -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterIntegrationTest "$1"
 }
 wait_ready() {
   attempt=0
@@ -20,10 +27,24 @@ wait_ready() {
   done
 }
 run_phase basic
+run_phase multipart-stage
+compose restart gateway
+wait_ready
+part_segment=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT segment_id FROM cluster_upload_segments LIMIT 1")
+printf '%s\n' "$part_segment" | grep -Eq '^[0-9a-f-]{36}$'
+part_shard=$(printf '%s' "$part_segment" | cut -c1-2)
+compose exec -T node-a sh -c 'printf corrupted > "/data/segments/$1/$2"' _ "$part_shard" "$part_segment"
+compose run --rm -T repair
+part_expected=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT encode(sha256,'hex') FROM cluster_upload_segments WHERE segment_id='$part_segment'")
+part_actual=$(compose exec -T node-a sha256sum "/data/segments/$part_shard/$part_segment" | cut -d' ' -f1)
+[ "$part_expected" = "$part_actual" ]
 run_phase same-host
 run_phase concurrent
 compose stop node-a
 run_phase degraded
+run_phase multipart-complete
 compose stop node-b
 run_phase quorum-lost
 compose start node-a node-b
@@ -40,6 +61,13 @@ expected=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
   "SELECT encode(s.sha256,'hex') FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE o.object_key='cluster-test/survivor' LIMIT 1")
 actual=$(compose exec -T node-a sha256sum "/data/segments/$shard/$segment_id" | cut -d' ' -f1)
 [ "$expected" = "$actual" ]
+compose exec -T metadata psql -U objectstore -d postgres -c \
+  'ALTER DATABASE objectstore SET default_transaction_read_only=on' >/dev/null
+status=$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:$host_port/ready")
+[ "$status" = 503 ]
+compose exec -T metadata psql -U objectstore -d postgres -c \
+  'ALTER DATABASE objectstore RESET default_transaction_read_only' >/dev/null
+wait_ready
 compose stop metadata
 status=$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:$host_port/ready")
 [ "$status" = 503 ]
@@ -55,4 +83,59 @@ compose up -d --no-deps gateway
 wait_ready
 run_phase recovered
 run_phase joined
+compose run --rm -T repair
+run_phase balanced
+export CLUSTER_MAINTENANCE_INTERVAL_SECONDS=1
+compose --profile automatic up -d maintenance
+node_a_id=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT node_id FROM cluster_nodes WHERE endpoint='http://node-a:9100'")
+segment_id=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT s.segment_id FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE '$node_a_id'::uuid = ANY(s.replica_ids) LIMIT 1")
+expected=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT encode(s.sha256,'hex') FROM cluster_segments s WHERE s.segment_id='$segment_id' LIMIT 1")
+shard=$(printf '%s' "$segment_id" | cut -c1-2)
+compose exec -T node-a sh -c 'printf corrupted > "/data/segments/$1/$2"' _ "$shard" "$segment_id"
+attempt=0
+while :; do
+  actual=$(compose exec -T node-a sha256sum "/data/segments/$shard/$segment_id" | cut -d' ' -f1)
+  [ "$actual" = "$expected" ] && break
+  attempt=$((attempt + 1))
+  [ "$attempt" -lt 90 ] || { echo 'Automatic repair did not restore the replica' >&2; exit 1; }
+  sleep 1
+done
+compose stop maintenance
+export CLUSTER_GC_TEST_MODE=true CLUSTER_GC_MIN_AGE_SECONDS=0
+compose stop node-c
+if gc_refusal=$(compose run --rm -T gc --apply 2>&1); then
+  echo 'Cleanup proceeded while replicas needed repair' >&2
+  exit 1
+fi
+printf '%s\n' "$gc_refusal" | grep -q 'Refusing cleanup while live segments need repair'
+compose start node-c
+before=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  'SELECT count(*) FROM cluster_gc_candidates')
+compose run --rm -T gc
+after=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  'SELECT count(*) FROM cluster_gc_candidates')
+[ "$before" = "$after" ]
+first_gc=$(compose run --rm -T gc --apply)
+printf '%s\n' "$first_gc" | grep -q '^orphan_candidates=[1-9]'
+printf '%s\n' "$first_gc" | grep -q '^segments_deleted=0$'
+second_gc=$(compose run --rm -T gc --apply)
+printf '%s\n' "$second_gc" | grep -q '^segments_deleted=[1-9]'
+run_phase recovered
+python3 scripts/test-cluster-http.py "$env_file" version-survivor
+run_phase verify-expanded
+backup_dir=$(mktemp -d)
+sh scripts/backup-cluster-metadata.sh "$env_file" "$backup_dir/metadata.dump"
+compose --profile recovery up -d --wait metadata-recovery
+compose exec -T metadata-recovery pg_restore -U objectstore -d objectstore --no-owner --no-acl \
+  < "$backup_dir/metadata.dump"
+compose stop metadata
+compose run --rm -T --no-deps \
+  -e 'POSTGRES_JDBC_URL=jdbc:postgresql://metadata:5432,metadata-recovery:5432/objectstore?connectTimeout=3&socketTimeout=10&targetServerType=primary&hostRecheckSeconds=0' \
+  --entrypoint java gateway --add-modules jdk.httpserver,java.net.http \
+  -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterIntegrationTest recovered
+compose start metadata
+wait_ready
 echo 'Cluster failure tests passed'

scripts/test-metadata.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+project="objectstore-metadata-test-$$"
+compose() { docker compose -p "$project" -f tests/metadata/compose.yaml "$@"; }
+trap 'compose down -v --remove-orphans >/dev/null 2>&1 || true' EXIT
+compose up --build --abort-on-container-exit --exit-code-from check check

scripts/test.sh

@@ -2,10 +2,14 @@
 set -eu
 cd "$(dirname "$0")/.."
 mkdir -p out/classes
-javac --release 21 --add-modules jdk.httpserver,java.net.http -d out/classes \
+javac --release 21 --add-modules jdk.httpserver,java.net.http -cp lib/hash4j-0.30.0.jar -d out/classes \
   src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java
-java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.StoreTest
+java --add-modules jdk.httpserver -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.StoreTest
-java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.ConcurrencyTest
+java --add-modules jdk.httpserver -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ConcurrencyTest
-java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.HttpTest
+java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.HttpTest
-java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.ClusterNodeTest
+java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClientLimitsTest
-java -cp out/classes cloud.lunarsky.store.CliTest
+java -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.EncryptedVolumeTest
+java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClusterNodeTest
+java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClusterTlsTest
+java -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.CliTest
+bash client/scripts/test.sh

src/cloud/lunarsky/store/Acl.java

@@ -0,0 +1,179 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.Headers;
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+final class Acl {
+    static final String ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers";
+    static final String AUTHENTICATED_USERS =
+        "http://acs.amazonaws.com/groups/global/AuthenticatedUsers";
+    static final int READ = 1;
+    static final int WRITE = 2;
+    static final int READ_ACP = 4;
+    static final int WRITE_ACP = 8;
+    static final int FULL_CONTROL = READ | WRITE | READ_ACP | WRITE_ACP;
+    private static final Map<String, Integer> PERMISSIONS = Map.of(
+        "READ", READ, "WRITE", WRITE, "READ_ACP", READ_ACP,
+        "WRITE_ACP", WRITE_ACP, "FULL_CONTROL", FULL_CONTROL);
+
+    private Acl() {}
+
+    static boolean allows(Map<String, String> grants, String principal, String owner, int permission) {
+        if (owner.equals(principal)) return true;
+        if (principal != null && (bits(grants.get(principal)) & permission) == permission) return true;
+        if (principal != null && (bits(grants.get(AUTHENTICATED_USERS)) & permission) == permission)
+            return true;
+        return (bits(grants.get(ALL_USERS)) & permission) == permission;
+    }
+
+    static void require(Map<String, String> grants, String principal, String owner, int permission) {
+        if (!allows(grants, principal, owner, permission))
+            throw new StoreException(403, "AccessDenied", "Access denied");
+    }
+
+    static Map<String, String> fromHeaders(Headers headers, Set<String> identities) {
+        String canned = SigV4.single(headers, "x-amz-acl");
+        Set<String> grantNames = Set.of("x-amz-grant-read", "x-amz-grant-write",
+            "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-grant-full-control");
+        for (String name : headers.keySet()) {
+            String lower = name.toLowerCase(java.util.Locale.ROOT);
+            if (lower.startsWith("x-amz-grant-") && !grantNames.contains(lower))
+                throw new StoreException(501, "NotImplemented", "Unsupported ACL grant header");
+            if (canned != null && grantNames.contains(lower))
+                throw new StoreException(400, "InvalidRequest", "Use either a canned ACL or explicit grants");
+        }
+        TreeMap<String, Integer> grants = new TreeMap<>();
+        if (canned != null) {
+            if (!canned.equals("private") && !canned.equals("public-read") &&
+                !canned.equals("authenticated-read") &&
+                !canned.equals("bucket-owner-full-control"))
+                throw new StoreException(400, "InvalidArgument", "Unsupported canned ACL");
+            if (canned.equals("public-read")) grants.put(ALL_USERS, READ);
+            if (canned.equals("authenticated-read")) grants.put(AUTHENTICATED_USERS, READ);
+        }
+        for (String name : new String[]{"read", "write", "read-acp", "write-acp", "full-control"}) {
+            String value = SigV4.single(headers, "x-amz-grant-" + name);
+            if (value == null) continue;
+            int permission = PERMISSIONS.get(name.replace('-', '_').toUpperCase(java.util.Locale.ROOT));
+            for (String grant : value.split(",")) {
+                String entry = grant.trim();
+                String principal;
+                if (entry.startsWith("id=\"") && entry.endsWith("\""))
+                    principal = entry.substring(4, entry.length() - 1);
+                else if (entry.startsWith("uri=\"") && entry.endsWith("\""))
+                    principal = entry.substring(5, entry.length() - 1);
+                else throw new StoreException(400, "InvalidArgument", "Invalid ACL grant");
+                if (!identities.contains(principal) && !principal.equals(ALL_USERS) &&
+                    !principal.equals(AUTHENTICATED_USERS))
+                    throw new StoreException(400, "InvalidArgument", "Unknown ACL grantee");
+                if (principal.equals(ALL_USERS) && permission != READ)
+                    throw new StoreException(400, "InvalidArgument", "Only public read is supported");
+                grants.merge(principal, permission, (left, right) -> left | right);
+            }
+        }
+        return encoded(grants);
+    }
+
+    static Map<String, String> validate(Map<String, String> values, Set<String> identities) {
+        if (values.size() > 64) throw new StoreException(400, "InvalidArgument", "Too many ACL grantees");
+        TreeMap<String, String> valid = new TreeMap<>();
+        for (var entry : values.entrySet()) {
+            String principal = entry.getKey();
+            int bits = bits(entry.getValue());
+            if ((!identities.contains(principal) && !principal.equals(ALL_USERS) &&
+                 !principal.equals(AUTHENTICATED_USERS)) ||
+                bits == 0 || (bits & ~FULL_CONTROL) != 0 ||
+                principal.equals(ALL_USERS) && bits != READ)
+                throw new StoreException(400, "InvalidArgument", "Invalid ACL grantee or permission");
+            valid.put(principal, Integer.toString(bits));
+        }
+        return Map.copyOf(valid);
+    }
+
+    static Map<String, String> fromXml(byte[] body, String owner, Set<String> identities) {
+        try {
+            var factory = javax.xml.parsers.DocumentBuilderFactory.newInstance();
+            factory.setNamespaceAware(true);
+            factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+            factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+            factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+            factory.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true);
+            factory.setExpandEntityReferences(false);
+            var document = factory.newDocumentBuilder().parse(new ByteArrayInputStream(body));
+            var root = document.getDocumentElement();
+            if (!root.getLocalName().equals("AccessControlPolicy")) throw new IllegalArgumentException();
+            var ownerNodes = root.getElementsByTagNameNS("*", "Owner");
+            var lists = root.getElementsByTagNameNS("*", "AccessControlList");
+            if (ownerNodes.getLength() != 1 || lists.getLength() != 1 ||
+                !owner.equals(text((org.w3c.dom.Element) ownerNodes.item(0), "ID")))
+                throw new IllegalArgumentException();
+            TreeMap<String, Integer> grants = new TreeMap<>();
+            var nodes = ((org.w3c.dom.Element) lists.item(0)).getElementsByTagNameNS("*", "Grant");
+            if (nodes.getLength() > 64) throw new IllegalArgumentException();
+            for (int i = 0; i < nodes.getLength(); i++) {
+                var grant = (org.w3c.dom.Element) nodes.item(i);
+                var grantees = grant.getElementsByTagNameNS("*", "Grantee");
+                if (grantees.getLength() != 1) throw new IllegalArgumentException();
+                var grantee = (org.w3c.dom.Element) grantees.item(0);
+                String type = grantee.getAttributeNS("http://www.w3.org/2001/XMLSchema-instance", "type");
+                String principal = switch (type) {
+                    case "CanonicalUser" -> text(grantee, "ID");
+                    case "Group" -> text(grantee, "URI");
+                    default -> throw new IllegalArgumentException();
+                };
+                Integer permission = PERMISSIONS.get(text(grant, "Permission"));
+                if (permission == null) throw new IllegalArgumentException();
+                grants.merge(principal, permission, (left, right) -> left | right);
+            }
+            grants.remove(owner);
+            return validate(encoded(grants), identities);
+        } catch (Exception error) {
+            throw new StoreException(400, "MalformedACLError", "Invalid access control policy");
+        }
+    }
+
+    private static String text(org.w3c.dom.Element element, String name) {
+        var nodes = element.getElementsByTagNameNS("*", name);
+        if (nodes.getLength() != 1) throw new IllegalArgumentException();
+        return nodes.item(0).getTextContent().trim();
+    }
+
+    static String xml(Map<String, String> grants, String owner) {
+        StringBuilder xml = new StringBuilder("<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>")
+            .append(owner).append("</ID></Owner><AccessControlList>");
+        grant(xml, owner, "FULL_CONTROL", false);
+        for (var entry : new TreeMap<>(grants).entrySet()) {
+            if (entry.getKey().equals(owner)) continue;
+            int bits = bits(entry.getValue());
+            if (bits == FULL_CONTROL) grant(xml, entry.getKey(), "FULL_CONTROL",
+                entry.getKey().equals(ALL_USERS) || entry.getKey().equals(AUTHENTICATED_USERS));
+            else for (var permission : PERMISSIONS.entrySet())
+                if (!permission.getKey().equals("FULL_CONTROL") && (bits & permission.getValue()) != 0)
+                    grant(xml, entry.getKey(), permission.getKey(),
+                        entry.getKey().equals(ALL_USERS) || entry.getKey().equals(AUTHENTICATED_USERS));
+        }
+        return xml.append("</AccessControlList></AccessControlPolicy>").toString();
+    }
+
+    private static void grant(StringBuilder xml, String principal, String permission, boolean group) {
+        xml.append("<Grant><Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"")
+            .append(group ? "Group\"><URI>" : "CanonicalUser\"><ID>")
+            .append(principal).append(group ? "</URI>" : "</ID>")
+            .append("</Grantee><Permission>").append(permission).append("</Permission></Grant>");
+    }
+
+    private static Map<String, String> encoded(Map<String, Integer> grants) {
+        TreeMap<String, String> values = new TreeMap<>();
+        grants.forEach((key, value) -> values.put(key, Integer.toString(value)));
+        return Map.copyOf(values);
+    }
+
+    private static int bits(String value) {
+        if (value == null) return 0;
+        try { return Integer.parseInt(value); }
+        catch (NumberFormatException error) { return 0; }
+    }
+}

src/cloud/lunarsky/store/AwsChunkedInputStream.java

@@ -0,0 +1,184 @@
+package cloud.lunarsky.store;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.Base64;
+import java.util.HexFormat;
+import java.util.zip.CRC32;
+import java.util.zip.CRC32C;
+import java.util.zip.Checksum;
+
+final class AwsChunkedInputStream extends FilterInputStream {
+    private static final String EMPTY_HASH = SigV4.hex(SigV4.hash(new byte[0]));
+    private final SigV4.Verified authorization;
+    private final long decodedLength;
+    private final String trailerName;
+    private final MessageDigest chunkHash;
+    private final MessageDigest trailerHash;
+    private final Checksum trailerCrc;
+    private final XxHashes trailerXxhash;
+    private long decoded;
+    private long chunkLeft;
+    private String suppliedSignature;
+    private String previousSignature;
+    private boolean finished;
+    private String trailerValue;
+
+    AwsChunkedInputStream(InputStream input, SigV4.Verified authorization,
+                          long decodedLength, String trailerName) {
+        super(input);
+        this.authorization = authorization;
+        this.decodedLength = decodedLength;
+        this.trailerName = trailerName;
+        this.previousSignature = authorization.signature();
+        try { this.chunkHash = MessageDigest.getInstance("SHA-256"); }
+        catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); }
+        this.trailerCrc = trailerName == null ? null : switch (trailerName) {
+            case "x-amz-checksum-crc32" -> new CRC32();
+            case "x-amz-checksum-crc32c" -> new CRC32C();
+            case "x-amz-checksum-crc64nvme" -> new Crc64Nvme();
+            default -> null;
+        };
+        this.trailerXxhash = trailerName != null && java.util.Set.of("x-amz-checksum-xxhash64",
+            "x-amz-checksum-xxhash3", "x-amz-checksum-xxhash128").contains(trailerName)
+            ? new XxHashes(trailerName.substring("x-amz-checksum-".length()).toUpperCase(java.util.Locale.ROOT))
+            : null;
+        String algorithm = trailerName == null ? null : switch (trailerName) {
+            case "x-amz-checksum-sha1" -> "SHA-1";
+            case "x-amz-checksum-sha256" -> "SHA-256";
+            case "x-amz-checksum-sha512" -> "SHA-512";
+            case "x-amz-checksum-md5" -> "MD5";
+            default -> null;
+        };
+        if (trailerName != null && trailerCrc == null && trailerXxhash == null && algorithm == null)
+            throw new StoreException(501, "NotImplemented", "Checksum trailer is unsupported");
+        try { this.trailerHash = algorithm == null ? null : MessageDigest.getInstance(algorithm); }
+        catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); }
+    }
+
+    String trailerValue() { return trailerValue; }
+
+    @Override public int read() throws IOException {
+        byte[] one = new byte[1];
+        int count;
+        do {
+            count = read(one, 0, 1);
+        } while (count == 0);
+        return count < 0 ? -1 : one[0] & 255;
+    }
+
+    @Override public int read(byte[] bytes, int offset, int length) throws IOException {
+        java.util.Objects.checkFromIndexSize(offset, length, bytes.length);
+        if (length == 0) return 0;
+        if (finished) return -1;
+        if (chunkLeft == 0) nextChunk();
+        if (finished) return -1;
+        int count = in.read(bytes, offset, (int) Math.min(length, chunkLeft));
+        if (count < 0) throw invalid("Incomplete signed chunk");
+        if (count == 0) return 0;
+        chunkHash.update(bytes, offset, count);
+        if (trailerCrc != null) trailerCrc.update(bytes, offset, count);
+        if (trailerHash != null) trailerHash.update(bytes, offset, count);
+        if (trailerXxhash != null) trailerXxhash.update(bytes, offset, count);
+        chunkLeft -= count;
+        decoded += count;
+        if (chunkLeft == 0) {
+            if (!line().isEmpty()) throw invalid("Missing signed chunk separator");
+            finishChunk();
+        }
+        return count;
+    }
+
+    private void nextChunk() throws IOException {
+        String line = line();
+        int separator = line.indexOf(";chunk-signature=");
+        if (separator < 1 || separator != line.lastIndexOf(";chunk-signature="))
+            throw invalid("Invalid signed chunk header");
+        String size = line.substring(0, separator);
+        suppliedSignature = line.substring(separator + 17);
+        if (!size.matches("[0-9a-fA-F]{1,16}") || !suppliedSignature.matches("[0-9a-f]{64}"))
+            throw invalid("Invalid signed chunk header");
+        try { chunkLeft = Long.parseUnsignedLong(size, 16); }
+        catch (NumberFormatException error) { throw invalid("Invalid signed chunk size"); }
+        if (chunkLeft > decodedLength - decoded) throw invalid("Signed chunks exceed decoded length");
+        chunkHash.reset();
+        if (chunkLeft == 0) finishPayload();
+    }
+
+    private void finishPayload() throws IOException {
+        finishChunk();
+        if (decoded != decodedLength) throw invalid("Decoded length mismatch");
+        if (trailerName == null) {
+            if (!line().isEmpty()) throw invalid("Invalid signed chunk ending");
+        } else {
+            verifyTrailer();
+        }
+        if (in.read() != -1) throw invalid("Extra bytes after signed payload");
+        finished = true;
+    }
+
+    private void verifyTrailer() throws IOException {
+        String trailer = line();
+        if (!trailer.startsWith(trailerName + ":")) throw invalid("Missing signed checksum trailer");
+        trailerValue = trailer.substring(trailerName.length() + 1);
+        if (!Base64.getEncoder().encodeToString(trailerChecksum()).equals(trailerValue))
+            throw new StoreException(400, "BadDigest", "Checksum trailer mismatch");
+        String signature = line();
+        if (!signature.matches("x-amz-trailer-signature=[0-9a-f]{64}"))
+            throw invalid("Missing trailer signature");
+        String toSign = "AWS4-HMAC-SHA256-TRAILER\n" + authorization.date() + "\n" +
+            authorization.scope() + "\n" + previousSignature + "\n" +
+            SigV4.hex(SigV4.hash((trailerName + ":" + trailerValue + "\n")
+                .getBytes(StandardCharsets.UTF_8)));
+        String expected = SigV4.hex(SigV4.hmac(authorization.signingKey(), toSign));
+        if (!MessageDigest.isEqual(expected.getBytes(StandardCharsets.US_ASCII),
+            signature.substring(24).getBytes(StandardCharsets.US_ASCII)))
+            throw invalid("Trailer signature mismatch");
+        if (!line().isEmpty()) throw invalid("Invalid trailer ending");
+    }
+
+    private byte[] trailerChecksum() {
+        if (trailerCrc == null)
+            return trailerXxhash != null ? trailerXxhash.digest() : trailerHash.digest();
+        long value = trailerCrc.getValue();
+        byte[] actual = new byte[trailerName.equals("x-amz-checksum-crc64nvme") ? 8 : 4];
+        for (int i = actual.length - 1; i >= 0; i--) {
+            actual[i] = (byte) value;
+            value >>>= 8;
+        }
+        return actual;
+    }
+
+    private void finishChunk() throws IOException {
+        String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + authorization.date() + "\n" +
+            authorization.scope() + "\n" + previousSignature + "\n" + EMPTY_HASH + "\n" +
+            SigV4.hex(chunkHash.digest());
+        byte[] expected = SigV4.hmac(authorization.signingKey(), toSign);
+        if (!MessageDigest.isEqual(expected, HexFormat.of().parseHex(suppliedSignature)))
+            throw invalid("Signed chunk signature mismatch");
+        previousSignature = suppliedSignature;
+    }
+
+    private String line() throws IOException {
+        byte[] bytes = new byte[512];
+        int count = 0;
+        while (count < bytes.length) {
+            int value = in.read();
+            if (value < 0) throw invalid("Incomplete signed chunk framing");
+            if (value == '\r') {
+                if (in.read() != '\n') throw invalid("Invalid signed chunk line ending");
+                return new String(bytes, 0, count, StandardCharsets.US_ASCII);
+            }
+            if (value < 32 || value > 126) throw invalid("Invalid signed chunk line");
+            bytes[count++] = (byte) value;
+        }
+        throw invalid("Signed chunk header is too long");
+    }
+
+    private static StoreException invalid(String message) {
+        return new StoreException(400, "InvalidRequest", message);
+    }
+}

src/cloud/lunarsky/store/Cli.java

@@ -79,6 +79,15 @@ public final class Cli {
         try (var paths = Files.walk(objects)) {
             for (Path path : paths.filter(Files::isRegularFile).toList()) inspectObject(objects, path, verify, report);
         }
+        Path versions = root.resolve("versions");
+        if (Files.isDirectory(versions)) {
+            try (var paths = Files.walk(versions)) {
+                for (Path path : paths.filter(Files::isRegularFile).toList()) {
+                    if (!path.getFileName().toString().equals("manifest"))
+                        inspectObject(versions, path, verify, report);
+                }
+            }
+        }
         Path multipart = root.resolve("multipart");
         if (Files.isDirectory(multipart)) {
             try (var uploads = Files.list(multipart)) {
@@ -100,7 +109,9 @@ public final class Cli {
             if (meta.key() == null) report.legacyObjects++;
             else {
                 String id = SigV4.hex(SigV4.hash((meta.bucket() + "/" + meta.key()).getBytes(StandardCharsets.UTF_8)));
-                Path expected = objects.resolve(id.substring(0, 2)).resolve(id);
+                Path expected = objects.getFileName().toString().equals("versions")
+                    ? objects.resolve(id.substring(0, 2)).resolve(id).resolve(path.getFileName())
+                    : objects.resolve(id.substring(0, 2)).resolve(id);
                 if (!path.equals(expected)) report.problem("Mismatched object path: " + path);
             }
             if (size - record.headerLength() != meta.length()) {
@@ -109,7 +120,9 @@ public final class Cli {
             }
             if (verify) {
                 MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
-                byte[] buffer = new byte[65536]; long count = 0; int n;
+                byte[] buffer = new byte[65536];
+                long count = 0;
+                int n;
                 while ((n = input.read(buffer)) != -1) {
                     count += n;
                     sha.update(buffer, 0, n);

src/cloud/lunarsky/store/ClientLimits.java

@@ -0,0 +1,253 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpExchange;
+import java.io.FilterInputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+final class ClientLimits {
+    private static final int MAX_CLIENTS = 10_000;
+    private static final long IDLE_NANOS = 300_000_000_000L;
+    private final int requestsPerSecond;
+    private final int requestBurst;
+    private final long bytesPerSecond;
+    private final long byteBurst;
+    private final int maxInFlight;
+    private final Set<String> trustedProxies;
+    private final Map<String, Client> clients = new HashMap<>();
+    private long admissions;
+
+    private ClientLimits(int requestsPerSecond, int requestBurst, long bytesPerSecond,
+                         long byteBurst, int maxInFlight, Set<String> trustedProxies) {
+        this.requestsPerSecond = requestsPerSecond;
+        this.requestBurst = requestBurst;
+        this.bytesPerSecond = bytesPerSecond;
+        this.byteBurst = byteBurst;
+        this.maxInFlight = maxInFlight;
+        this.trustedProxies = trustedProxies;
+    }
+
+    static ClientLimits disabled() {
+        return new ClientLimits(0, 0, 0, 0, 0, Set.of());
+    }
+
+    static ClientLimits fromEnvironment(Map<String, String> environment) {
+        int requests = number(environment, "PUBLIC_REQUESTS_PER_SECOND", 0);
+        int requestBurst = number(environment, "PUBLIC_REQUEST_BURST", requests);
+        long bytes = longNumber(environment, "PUBLIC_BYTES_PER_SECOND", 0);
+        long byteBurst = longNumber(environment, "PUBLIC_BYTE_BURST", bytes);
+        int inFlight = number(environment, "PUBLIC_MAX_IN_FLIGHT_PER_IP",
+            requests > 0 || bytes > 0 ? 8 : 0);
+        if (requests < 0 || requestBurst < 0 || bytes < 0 || byteBurst < 0 || inFlight < 0 ||
+            requests > 0 && requestBurst < 1 || requests == 0 && requestBurst != 0 ||
+            bytes > 0 && (byteBurst < 1 || byteBurst > 1_073_741_824L) ||
+            bytes == 0 && byteBurst != 0 ||
+            (requests > 0 || bytes > 0) && inFlight < 1)
+            throw new IllegalArgumentException("Invalid public client limits");
+        Set<String> proxies = new HashSet<>();
+        String configured = environment.getOrDefault("PUBLIC_TRUSTED_PROXY_IPS", "").trim();
+        if (!configured.isEmpty()) {
+            for (String item : configured.split(",", -1))
+                proxies.add(numericAddress(item.trim()).getHostAddress());
+        }
+        if (requests == 0 && bytes == 0 && inFlight == 0 && !proxies.isEmpty())
+            throw new IllegalArgumentException("Trusted proxy IPs require public client limits");
+        return new ClientLimits(requests, requestBurst, bytes, byteBurst, inFlight, Set.copyOf(proxies));
+    }
+
+    private static int number(Map<String, String> environment, String name, int fallback) {
+        String value = environment.get(name);
+        if (value == null || value.isBlank()) return fallback;
+        try { return Integer.parseInt(value); }
+        catch (NumberFormatException error) { throw new IllegalArgumentException("Invalid " + name, error); }
+    }
+
+    private static long longNumber(Map<String, String> environment, String name, long fallback) {
+        String value = environment.get(name);
+        if (value == null || value.isBlank()) return fallback;
+        try { return Long.parseLong(value); }
+        catch (NumberFormatException error) { throw new IllegalArgumentException("Invalid " + name, error); }
+    }
+
+    private static InetAddress numericAddress(String value) {
+        try {
+            if (value.matches("[0-9]{1,3}(\\.[0-9]{1,3}){3}")) {
+                String[] parts = value.split("\\.");
+                byte[] octets = new byte[4];
+                for (int i = 0; i < 4; i++) {
+                    int octet = Integer.parseInt(parts[i]);
+                    if (octet > 255) throw new IllegalArgumentException("Invalid IP address");
+                    octets[i] = (byte) octet;
+                }
+                return InetAddress.getByAddress(octets);
+            }
+            if (value.contains(":") && value.matches("[0-9A-Fa-f:.]+"))
+                return InetAddress.getByName(value);
+        } catch (UnknownHostException error) {
+            throw new IllegalArgumentException("Invalid IP address", error);
+        }
+        throw new IllegalArgumentException("Expected numeric IP address");
+    }
+
+    private String address(HttpExchange exchange) {
+        InetAddress peer = exchange.getRemoteAddress().getAddress();
+        String peerAddress = peer.getHostAddress();
+        if (!trustedProxies.contains(peerAddress)) return peerAddress;
+        var values = exchange.getRequestHeaders().get("X-Real-IP");
+        if (values == null || values.size() != 1)
+            throw new StoreException(400, "InvalidRequest", "Trusted proxy must supply one X-Real-IP address");
+        try { return numericAddress(values.getFirst()).getHostAddress(); }
+        catch (IllegalArgumentException error) {
+            throw new StoreException(400, "InvalidRequest", "Trusted proxy supplied an invalid client address");
+        }
+    }
+
+    Client enter(HttpExchange exchange) {
+        if (requestsPerSecond == 0 && bytesPerSecond == 0 && maxInFlight == 0) return null;
+        String path = exchange.getRequestURI().getRawPath();
+        if (exchange.getRemoteAddress().getAddress().isLoopbackAddress() &&
+            exchange.getRequestHeaders().get("X-Real-IP") == null &&
+            path.equals("/health")) return null;
+        Client client = admit(address(exchange));
+        if (bytesPerSecond > 0) {
+            try {
+                exchange.setStreams(new LimitedInput(exchange.getRequestBody(), client),
+                    new LimitedOutput(exchange.getResponseBody(), client));
+            } catch (RuntimeException error) {
+                leave(client);
+                throw error;
+            }
+        }
+        return client;
+    }
+
+    private synchronized Client admit(String address) {
+        Client client;
+        long now = System.nanoTime();
+        if (++admissions % 1024 == 0 || clients.size() >= MAX_CLIENTS)
+            clients.entrySet().removeIf(entry -> entry.getValue().inFlight == 0 &&
+                now - entry.getValue().lastSeen > IDLE_NANOS);
+        client = clients.get(address);
+        if (client == null) {
+            if (clients.size() >= MAX_CLIENTS)
+                throw new StoreException(503, "SlowDown", "Client limit table is full");
+            client = new Client(now, requestBurst, byteBurst);
+            clients.put(address, client);
+        }
+        refill(client, now);
+        client.lastSeen = now;
+        if (maxInFlight > 0 && client.inFlight >= maxInFlight)
+            throw new StoreException(503, "SlowDown", "Too many concurrent requests from this client");
+        if (requestsPerSecond > 0 && client.requestTokens < 1)
+            throw new StoreException(503, "SlowDown", "Client request rate exceeded");
+        if (requestsPerSecond > 0) client.requestTokens--;
+        client.inFlight++;
+        return client;
+    }
+
+    synchronized void leave(Client client) {
+        if (client != null) {
+            client.inFlight--;
+            client.lastSeen = System.nanoTime();
+        }
+    }
+
+    private void refill(Client client, long now) {
+        double seconds = Math.max(0, now - client.lastRefill) / 1_000_000_000.0;
+        if (requestsPerSecond > 0)
+            client.requestTokens = Math.min(requestBurst, client.requestTokens + seconds * requestsPerSecond);
+        if (bytesPerSecond > 0)
+            client.byteTokens = Math.min(byteBurst, client.byteTokens + seconds * bytesPerSecond);
+        client.lastRefill = now;
+    }
+
+    private void pace(Client client, int count) throws IOException {
+        while (true) {
+            long wait;
+            synchronized (this) {
+                refill(client, System.nanoTime());
+                if (client.byteTokens >= count) {
+                    client.byteTokens -= count;
+                    return;
+                }
+                wait = Math.max(1_000_000L,
+                    (long) Math.ceil((count - client.byteTokens) * 1_000_000_000.0 / bytesPerSecond));
+            }
+            try { Thread.sleep(Math.min(wait / 1_000_000L + 1, 1000)); }
+            catch (InterruptedException error) {
+                Thread.currentThread().interrupt();
+                throw new IOException("Transfer interrupted while waiting for client bandwidth", error);
+            }
+        }
+    }
+
+    private int chunk() { return (int) Math.min(16_384, byteBurst); }
+
+    static final class Client {
+        private long lastSeen;
+        private long lastRefill;
+        private double requestTokens;
+        private double byteTokens;
+        private int inFlight;
+
+        private Client(long now, int requestBurst, long byteBurst) {
+            lastSeen = now;
+            lastRefill = now;
+            requestTokens = requestBurst;
+            byteTokens = byteBurst;
+        }
+    }
+
+    private final class LimitedInput extends FilterInputStream {
+        private final Client client;
+
+        private LimitedInput(java.io.InputStream input, Client client) {
+            super(input);
+            this.client = client;
+        }
+
+        @Override public int read() throws IOException {
+            int value = in.read();
+            if (value >= 0) pace(client, 1);
+            return value;
+        }
+
+        @Override public int read(byte[] bytes, int offset, int length) throws IOException {
+            int count = in.read(bytes, offset, Math.min(length, chunk()));
+            if (count > 0) pace(client, count);
+            return count;
+        }
+    }
+
+    private final class LimitedOutput extends FilterOutputStream {
+        private final Client client;
+
+        private LimitedOutput(java.io.OutputStream output, Client client) {
+            super(output);
+            this.client = client;
+        }
+
+        @Override public void write(int value) throws IOException {
+            pace(client, 1);
+            out.write(value);
+        }
+
+        @Override public void write(byte[] bytes, int offset, int length) throws IOException {
+            java.util.Objects.checkFromIndexSize(offset, length, bytes.length);
+            int left = length;
+            while (left > 0) {
+                int count = Math.min(left, chunk());
+                pace(client, count);
+                out.write(bytes, offset, count);
+                offset += count;
+                left -= count;
+            }
+        }
+    }
+}

src/cloud/lunarsky/store/ClusterGc.java

@@ -0,0 +1,45 @@
+package cloud.lunarsky.store;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Map;
+
+public final class ClusterGc {
+    public static void main(String[] args) throws Exception {
+        if (args.length > 1 || (args.length == 1 && !args[0].equals("--apply")))
+            throw new IllegalArgumentException("Usage: objectstore cluster-gc [--apply]");
+        run(System.getenv(), args.length == 1);
+    }
+
+    static void run(Map<String, String> env, boolean apply) throws Exception {
+        if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV")))
+            throw new IllegalArgumentException("Cluster garbage collection is only enabled in local cluster mode");
+        boolean testDomains = "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS"));
+        ObjectStorage.Limits limits = StorageLimits.fromEnvironment(env);
+        boolean disposableTest = testDomains && "true".equals(env.get("CLUSTER_GC_TEST_MODE"));
+        long age = Long.parseLong(env.getOrDefault("CLUSTER_GC_MIN_AGE_SECONDS", "1209600"));
+        long backupRetention = Long.parseLong(env.getOrDefault("CLUSTER_BACKUP_RETENTION_SECONDS", "0"));
+        if (age < 0 || age > 315360000 || (age == 0 && !disposableTest))
+            throw new IllegalArgumentException("Invalid garbage collection age");
+        if (apply && !disposableTest && (backupRetention < 86400 || age <= backupRetention))
+            throw new IllegalArgumentException("Set a garbage collection age longer than the backup retention");
+        try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"),
+                env.get("POSTGRES_PASSWORD"), env.get("S3_BUCKET"),
+                Arrays.stream(env.get("CLUSTER_NODES").split(",")).map(URI::create).toList(),
+                env.get("CLUSTER_TOKEN"), env.get("CLUSTER_REPAIR_TOKEN"),
+                limits.maxObjectBytes(), limits.maxTotalBytes(),
+                testDomains)) {
+            if (apply) {
+                var repair = store.repairOnce();
+                if (repair.underReplicated() > 0 || repair.unrecoverable() > 0)
+                    throw new IllegalStateException("Refusing cleanup while live segments need repair");
+            }
+            var report = store.collectGarbage(age * 1000, apply);
+            System.out.println("segments_scanned=" + report.scanned());
+            System.out.println("orphan_candidates=" + report.eligible());
+            System.out.println("segments_deleted=" + report.deleted());
+            System.out.println("unavailable_nodes=" + report.unavailableNodes());
+            if (report.unavailableNodes() > 0) throw new IllegalStateException("Cleanup did not scan every node");
+        }
+    }
+}

src/cloud/lunarsky/store/ClusterJoin.java

@@ -12,6 +12,8 @@ public final class ClusterJoin {
         if (args.length != 2)
             throw new IllegalArgumentException("Usage: objectstore cluster-join node-url expected-host-uuid");
         Map<String, String> env = System.getenv();
+        EncryptedVolume.requireConfigured(env,
+            java.nio.file.Path.of(System.getProperty("java.io.tmpdir")));
         if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV")))
             throw new IllegalArgumentException("Node registration is only enabled in local cluster mode");
         URI url = URI.create(args[0]);

src/cloud/lunarsky/store/ClusterMigrate.java

@@ -24,6 +24,8 @@ public final class ClusterMigrate {
             (args.length != 2 || !args[0].equals("--apply")))
             throw new IllegalArgumentException("Usage: objectstore cluster-migrate --check | --apply node-id-0,node-id-1,node-id-2");
         Map<String, String> env = System.getenv();
+        EncryptedVolume.requireConfigured(env,
+            java.nio.file.Path.of(System.getProperty("java.io.tmpdir")));
         if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV")))
             throw new IllegalArgumentException("Migration is enabled only in local cluster mode");
         List<URI> urls = Arrays.stream(env.get("CLUSTER_NODES").split(",", -1)).map(URI::create).toList();
@@ -148,7 +150,8 @@ public final class ClusterMigrate {
             }
             connection.commit();
         } catch (SQLException | IOException | RuntimeException error) {
-            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
+            try { connection.rollback(); }
+            catch (SQLException rollback) { error.addSuppressed(rollback); }
             if (error instanceof IOException io) throw io;
             if (error instanceof SQLException sql) throw sql;
             throw (RuntimeException) error;

src/cloud/lunarsky/store/ClusterNode.java

@@ -1,7 +1,6 @@
 package cloud.lunarsky.store;
 
 import com.sun.net.httpserver.HttpExchange;
-import com.sun.net.httpserver.HttpServer;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -15,7 +14,9 @@ import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.security.MessageDigest;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HexFormat;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Executors;
@@ -31,6 +32,7 @@ public final class ClusterNode implements AutoCloseable {
     private final FileLock lock;
 
     ClusterNode(Path root, String token, String repairToken, UUID hostId) throws IOException {
+        EncryptedVolume.requireConfigured(System.getenv(), root);
         if (token == null || token.length() < 32) throw new IllegalArgumentException("Cluster token must have at least 32 characters");
         if (repairToken == null || repairToken.length() < 32 || repairToken.equals(token))
             throw new IllegalArgumentException("A separate repair token of at least 32 characters is required");
@@ -78,43 +80,24 @@ public final class ClusterNode implements AutoCloseable {
                 respond(exchange, 200, "ok");
                 return;
             }
-            byte[] supplied = exchange.getRequestHeaders().getFirst("X-Cluster-Token") == null
+            if (!authorized(exchange)) return;
-                ? new byte[0] : exchange.getRequestHeaders().getFirst("X-Cluster-Token")
-                    .getBytes(java.nio.charset.StandardCharsets.UTF_8);
-            if (!MessageDigest.isEqual(token, supplied)) {
-                respond(exchange, 403, "Forbidden");
-                return;
-            }
             if (path.equals("/identity") && exchange.getRequestMethod().equals("GET")) {
                 respond(exchange, 200, identity.nodeId() + " " + identity.hostId());
                 return;
             }
-            if (!identity.nodeId().toString().equals(exchange.getRequestHeaders().getFirst("X-Cluster-Expected-Node"))) {
+            if (!expectedNodeAndRepairAuthorized(exchange)) return;
-                respond(exchange, 409, "Wrong storage node");
+            if (path.equals("/segments") && exchange.getRequestMethod().equals("GET")) {
-                return;
+                if (maintenanceAuthorized(exchange)) inventory(exchange);
-            }
-            if (exchange.getRequestMethod().equals("PUT") &&
-                "true".equals(exchange.getRequestHeaders().getFirst("X-Cluster-Repair"))) {
-                String suppliedRepair = exchange.getRequestHeaders().getFirst("X-Cluster-Repair-Token");
-                byte[] suppliedBytes = suppliedRepair == null ? new byte[0]
-                    : suppliedRepair.getBytes(java.nio.charset.StandardCharsets.UTF_8);
-                if (!MessageDigest.isEqual(repairToken, suppliedBytes)) {
-                    respond(exchange, 403, "Repair authority required");
-                    return;
-                }
-            }
-            if (!path.matches("/segments/[0-9a-f-]{36}")) {
-                respond(exchange, 404, "Not found");
-                return;
-            }
-            String id = path.substring("/segments/".length());
-            if (!UUID.fromString(id).toString().equals(id)) {
-                respond(exchange, 400, "Invalid segment ID");
                 return;
             }
+            String id = segmentId(exchange, path);
+            if (id == null) return;
             switch (exchange.getRequestMethod()) {
                 case "PUT" -> put(exchange, segmentPath(id, true));
                 case "GET" -> get(exchange, segmentPath(id, false));
+                case "DELETE" -> {
+                    if (maintenanceAuthorized(exchange)) delete(exchange, segmentPath(id, false));
+                }
                 default -> respond(exchange, 405, "Method not allowed");
             }
         } catch (IllegalArgumentException error) {
@@ -127,6 +110,119 @@ public final class ClusterNode implements AutoCloseable {
         }
     }
 
+    private boolean authorized(HttpExchange exchange) throws IOException {
+        byte[] supplied = exchange.getRequestHeaders().getFirst("X-Cluster-Token") == null
+            ? new byte[0] : exchange.getRequestHeaders().getFirst("X-Cluster-Token")
+                .getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        if (!MessageDigest.isEqual(token, supplied)) {
+            respond(exchange, 403, "Forbidden");
+            return false;
+        }
+        return true;
+    }
+
+    private boolean expectedNodeAndRepairAuthorized(HttpExchange exchange) throws IOException {
+        if (!identity.nodeId().toString().equals(exchange.getRequestHeaders().getFirst("X-Cluster-Expected-Node"))) {
+            respond(exchange, 409, "Wrong storage node");
+            return false;
+        }
+        if (exchange.getRequestMethod().equals("PUT") &&
+            "true".equals(exchange.getRequestHeaders().getFirst("X-Cluster-Repair"))) {
+            String suppliedRepair = exchange.getRequestHeaders().getFirst("X-Cluster-Repair-Token");
+            byte[] suppliedBytes = suppliedRepair == null ? new byte[0]
+                : suppliedRepair.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+            if (!MessageDigest.isEqual(repairToken, suppliedBytes)) {
+                respond(exchange, 403, "Repair authority required");
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean maintenanceAuthorized(HttpExchange exchange) throws IOException {
+        String supplied = exchange.getRequestHeaders().getFirst("X-Cluster-Repair-Token");
+        byte[] value = supplied == null ? new byte[0] : supplied.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        if (MessageDigest.isEqual(repairToken, value)) return true;
+        respond(exchange, 403, "Repair authority required");
+        return false;
+    }
+
+    private void inventory(HttpExchange exchange) throws IOException {
+        String query = exchange.getRequestURI().getRawQuery();
+        if (query == null || !query.matches("shard=[0-9a-f]{2}(&after=[0-9a-f-]{36})?")) {
+            respond(exchange, 400, "Invalid inventory request");
+            return;
+        }
+        String shard = query.substring(6, 8);
+        String after = query.length() > 8 ? query.substring(15) : "";
+        if (!after.isEmpty() && (!after.startsWith(shard) || !UUID.fromString(after).toString().equals(after))) {
+            respond(exchange, 400, "Invalid inventory cursor");
+            return;
+        }
+        Path directory = segments.resolve(shard);
+        if (!Files.isDirectory(directory)) {
+            respond(exchange, 200, "");
+            return;
+        }
+        List<Path> files;
+        try (var entries = Files.list(directory)) {
+            files = entries.filter(Files::isRegularFile).sorted(Comparator.comparing(path ->
+                path.getFileName().toString())).toList();
+        }
+        StringBuilder body = new StringBuilder();
+        int count = 0;
+        for (Path file : files) {
+            String id = file.getFileName().toString();
+            if (id.compareTo(after) <= 0 || !id.matches("[0-9a-f-]{36}")) continue;
+            if (!UUID.fromString(id).toString().equals(id)) continue;
+            body.append(id).append(' ').append(Files.getLastModifiedTime(file).toMillis()).append('\n');
+            if (++count == 1000) break;
+        }
+        respond(exchange, 200, body.toString());
+    }
+
+    private synchronized void delete(HttpExchange exchange, Path target) throws IOException {
+        String expected = exchange.getRequestHeaders().getFirst("X-Cluster-Expected-Mtime");
+        String age = exchange.getRequestHeaders().getFirst("X-Cluster-Gc-Min-Age-Millis");
+        long expectedTime, minimumAge;
+        try {
+            expectedTime = Long.parseLong(expected);
+            minimumAge = Long.parseLong(age);
+        } catch (NumberFormatException error) {
+            respond(exchange, 400, "Invalid deletion guard");
+            return;
+        }
+        if (minimumAge < 0 || minimumAge > System.currentTimeMillis()) {
+            respond(exchange, 400, "Invalid deletion age");
+            return;
+        }
+        if (!Files.isRegularFile(target)) {
+            exchange.sendResponseHeaders(404, -1);
+            return;
+        }
+        long modified = Files.getLastModifiedTime(target).toMillis();
+        if (modified != expectedTime || modified > System.currentTimeMillis() - minimumAge) {
+            exchange.sendResponseHeaders(409, -1);
+            return;
+        }
+        Files.delete(target);
+        DiskStore.syncDirectory(target.getParent());
+        exchange.sendResponseHeaders(204, -1);
+    }
+
+    private static String segmentId(HttpExchange exchange, String path) throws IOException {
+        if (!path.matches("/segments/[0-9a-f-]{36}")) {
+            respond(exchange, 404, "Not found");
+            return null;
+        }
+        String id = path.substring("/segments/".length());
+        if (!UUID.fromString(id).toString().equals(id)) {
+            respond(exchange, 400, "Invalid segment ID");
+            return null;
+        }
+        return id;
+    }
+
     private synchronized Path segmentPath(String id, boolean createShard) throws IOException {
         Path shard = segments.resolve(id.substring(0, 2));
         if (createShard && !Files.isDirectory(shard)) {
@@ -229,16 +325,19 @@ public final class ClusterNode implements AutoCloseable {
             env.get("CLUSTER_REPAIR_TOKEN"),
             UUID.fromString(env.get("CLUSTER_HOST_ID")));
         int port = Integer.parseInt(env.getOrDefault("NODE_PORT", "9100"));
-        var server = HttpServer.create(new InetSocketAddress(env.getOrDefault("NODE_BIND", "127.0.0.1"), port), 64);
+        var server = ClusterTls.nodeServer(
+            new InetSocketAddress(env.getOrDefault("NODE_BIND", "127.0.0.1"), port), env);
         var executor = Executors.newVirtualThreadPerTaskExecutor();
         server.setExecutor(executor);
         server.createContext("/", node::handle);
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             server.stop(5);
             executor.close();
-            try { node.close(); } catch (IOException error) { System.err.println("Node close failed: " + error); }
+            try { node.close(); }
+            catch (IOException error) { System.err.println("Node close failed: " + error); }
         }));
         server.start();
-        System.out.println("ObjectStore cluster node listening on :" + port);
+        System.out.println("ObjectStore cluster node listening on :" + port +
+            (server instanceof com.sun.net.httpserver.HttpsServer ? " (TLS)" : " (HTTP)"));
     }
 }

src/cloud/lunarsky/store/ClusterRepair.java

@@ -6,21 +6,43 @@ import java.util.Map;
 
 public final class ClusterRepair {
     public static void main(String[] args) throws Exception {
-        if (args.length != 0) throw new IllegalArgumentException("Usage: objectstore cluster-repair");
+        if (args.length > 1 || (args.length == 1 && !args[0].equals("--loop")))
+            throw new IllegalArgumentException("Usage: objectstore cluster-repair [--loop]");
         Map<String, String> env = System.getenv();
         if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV")))
             throw new IllegalArgumentException("Cluster repair is only enabled in local cluster mode");
+        boolean loop = args.length == 1;
+        ObjectStorage.Limits limits = StorageLimits.fromEnvironment(env);
+        long seconds = Long.parseLong(env.getOrDefault("CLUSTER_MAINTENANCE_INTERVAL_SECONDS", "60"));
+        if (seconds < 1 || seconds > 3600) throw new IllegalArgumentException("Invalid maintenance interval");
+        boolean gcEnabled = loop && "true".equals(env.get("CLUSTER_GC_ENABLED"));
+        long gcInterval = Long.parseLong(env.getOrDefault("CLUSTER_GC_INTERVAL_SECONDS", "86400"));
+        if (gcEnabled && (gcInterval < 1 || gcInterval > 604800))
+            throw new IllegalArgumentException("Invalid garbage collection interval");
+        long nextGc = 0;
+        do {
             try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"),
                     env.get("POSTGRES_PASSWORD"), env.get("S3_BUCKET"),
                     Arrays.stream(env.get("CLUSTER_NODES").split(",")).map(URI::create).toList(),
-                env.get("CLUSTER_TOKEN"), env.get("CLUSTER_REPAIR_TOKEN"), 134217728, 2147483648L,
+                    env.get("CLUSTER_TOKEN"), env.get("CLUSTER_REPAIR_TOKEN"),
+                    limits.maxObjectBytes(), limits.maxTotalBytes(),
                     "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS")))) {
                 var report = store.repairOnce();
                 System.out.println("segments_scanned=" + report.scanned());
                 System.out.println("replicas_restored=" + report.restored());
+                System.out.println("segments_rebalanced=" + report.rebalanced());
                 System.out.println("segments_under_replicated=" + report.underReplicated());
                 System.out.println("segments_unrecoverable=" + report.unrecoverable());
-            if (report.unrecoverable() > 0) System.exit(1);
+                if (!loop && report.unrecoverable() > 0) System.exit(1);
+                if (gcEnabled && System.currentTimeMillis() >= nextGc) {
+                    ClusterGc.run(env, true);
+                    nextGc = System.currentTimeMillis() + gcInterval * 1000;
                 }
+            } catch (Exception error) {
+                if (!loop) throw error;
+                System.err.println("Cluster maintenance failed: " + error.getMessage());
+            }
+            if (loop) Thread.sleep(seconds * 1000);
+        } while (loop);
     }
 }

src/cloud/lunarsky/store/ClusterTls.java

@@ -0,0 +1,99 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpServer;
+import com.sun.net.httpserver.HttpsConfigurator;
+import com.sun.net.httpserver.HttpsServer;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.net.http.HttpClient;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Map;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+
+final class ClusterTls {
+    private ClusterTls() { }
+
+    static HttpServer nodeServer(InetSocketAddress address, Map<String, String> env) throws IOException {
+        String keyStore = env.get("NODE_TLS_KEYSTORE");
+        String passwordFile = env.get("NODE_TLS_PASSWORD_FILE");
+        if (missing(keyStore) && missing(passwordFile)) return HttpServer.create(address, 64);
+        if (missing(keyStore) || missing(passwordFile))
+            throw new IOException("Node TLS requires both NODE_TLS_KEYSTORE and NODE_TLS_PASSWORD_FILE");
+        SSLContext context = serverContext(Path.of(keyStore), Path.of(passwordFile));
+        HttpsServer server = HttpsServer.create(address, 64);
+        server.setHttpsConfigurator(new HttpsConfigurator(context));
+        return server;
+    }
+
+    static HttpClient client(Map<String, String> env, Duration timeout) throws IOException {
+        String trustStore = env.get("CLUSTER_TLS_TRUSTSTORE");
+        String passwordFile = env.get("CLUSTER_TLS_PASSWORD_FILE");
+        if (missing(trustStore) != missing(passwordFile))
+            throw new IOException("Cluster TLS requires both CLUSTER_TLS_TRUSTSTORE and CLUSTER_TLS_PASSWORD_FILE");
+        HttpClient.Builder builder = HttpClient.newBuilder().connectTimeout(timeout);
+        if (!missing(trustStore))
+            builder.sslContext(clientContext(Path.of(trustStore), Path.of(passwordFile)));
+        return builder.build();
+    }
+
+    private static SSLContext serverContext(Path keyStore, Path passwordFile) throws IOException {
+        char[] password = password(passwordFile);
+        try {
+            KeyStore keys = load(keyStore, password);
+            KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+            managers.init(keys, password);
+            SSLContext context = SSLContext.getInstance("TLS");
+            context.init(managers.getKeyManagers(), null, null);
+            return context;
+        } catch (GeneralSecurityException error) {
+            throw new IOException("Could not configure node TLS", error);
+        } finally {
+            Arrays.fill(password, '\0');
+        }
+    }
+
+    private static SSLContext clientContext(Path trustStore, Path passwordFile) throws IOException {
+        char[] password = password(passwordFile);
+        try {
+            KeyStore trust = load(trustStore, password);
+            TrustManagerFactory managers = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            managers.init(trust);
+            SSLContext context = SSLContext.getInstance("TLS");
+            context.init(null, managers.getTrustManagers(), null);
+            return context;
+        } catch (GeneralSecurityException error) {
+            throw new IOException("Could not configure cluster TLS trust", error);
+        } finally {
+            Arrays.fill(password, '\0');
+        }
+    }
+
+    private static KeyStore load(Path path, char[] password) throws IOException, GeneralSecurityException {
+        KeyStore store = KeyStore.getInstance("PKCS12");
+        try (InputStream input = Files.newInputStream(path)) {
+            store.load(input, password);
+        }
+        return store;
+    }
+
+    private static char[] password(Path file) throws IOException {
+        String value = Files.readString(file, StandardCharsets.UTF_8);
+        if (value.endsWith("\n")) value = value.substring(0, value.length() - 1);
+        if (value.endsWith("\r")) value = value.substring(0, value.length() - 1);
+        if (value.isEmpty()) throw new IOException("Cluster TLS password file is empty");
+        return value.toCharArray();
+    }
+
+    private static boolean missing(String value) {
+        return value == null || value.isBlank();
+    }
+}

src/cloud/lunarsky/store/Crc64Nvme.java

@@ -0,0 +1,39 @@
+package cloud.lunarsky.store;
+
+import java.util.zip.Checksum;
+import java.util.Base64;
+import java.nio.ByteBuffer;
+
+final class Crc64Nvme implements Checksum {
+    private static final long POLYNOMIAL = 0x9a6c9329ac4bc9b5L;
+    private static final long[] TABLE = table();
+    private long state = -1L;
+
+    private static long[] table() {
+        long[] values = new long[256];
+        for (int index = 0; index < values.length; index++) {
+            long value = index;
+            for (int bit = 0; bit < 8; bit++)
+                value = (value >>> 1) ^ ((value & 1L) == 0 ? 0 : POLYNOMIAL);
+            values[index] = value;
+        }
+        return values;
+    }
+
+    @Override public void update(int value) {
+        state = (state >>> 8) ^ TABLE[(int) (state ^ value) & 255];
+    }
+
+    @Override public void update(byte[] bytes, int offset, int length) {
+        java.util.Objects.checkFromIndexSize(offset, length, bytes.length);
+        for (int i = offset; i < offset + length; i++) update(bytes[i]);
+    }
+
+    @Override public long getValue() { return ~state; }
+
+    String encoded() {
+        return Base64.getEncoder().encodeToString(ByteBuffer.allocate(8).putLong(getValue()).array());
+    }
+
+    @Override public void reset() { state = -1L; }
+}

src/cloud/lunarsky/store/DiskStore.java

@@ -19,23 +19,43 @@ import cloud.lunarsky.store.ObjectStorage.ListPage;
 final class DiskStore implements ObjectStorage {
     private static final long MAGIC_V1 = 0x4c534f424a303031L;
     private static final long MAGIC_V2 = 0x4c534f424a303032L;
+    private static final long MAGIC_V3 = 0x4c534f424a303033L;
+    private static final long MAGIC_V4 = 0x4c534f424a303034L;
+    private static final long MAGIC_V5 = 0x4c534f424a303035L;
     private static final int HEADER_V1 = 72;
     private static final int HEADER_V2 = 78;
-    private final Path root, objects, temporary;
+    private static final int HEADER_V3 = 82;
+    private static final int CHECKSUM_AREA = 512;
+    private static final int HEADER_V4 = HEADER_V3 + 2 + CHECKSUM_AREA;
+    private static final int ACL_AREA = 2048;
+    private static final int HEADER_V5 = HEADER_V4 + 2;
+    private static final int BUCKET_MAGIC = 0x4c534243;
+    private static final int BUCKET_MAGIC_V2 = 0x4c534244;
+    private static final int BUCKET_MAGIC_V3 = 0x4c534245;
+    private static final int VERSION_MAGIC = 0x4c53564d;
+    private final Path root, objects, temporary, catalog, versions;
     private final FileChannel lockChannel;
     private final FileLock processLock;
     private final long maxObject, maxTotal;
     private final Object[] locks = new Object[128];
     private final NavigableMap<String, Metadata> index = new TreeMap<>();
+    private final NavigableMap<String, Bucket> buckets = new TreeMap<>();
+    private final NavigableMap<String, List<VersionRecord>> histories = new TreeMap<>();
     private long used;
     private long objectCount, legacyCount;
 
     record Record(Metadata metadata, int headerLength) {}
+    private record VersionRecord(String id, String storageId, boolean marker, long modified) {}
 
     DiskStore(Path root, long maxObject, long maxTotal) throws IOException {
+        EncryptedVolume.requireConfigured(System.getenv(), root);
         this.root = root;
-        objects = root.resolve("objects"); temporary = root.resolve("pending");
+        objects = root.resolve("objects");
-        this.maxObject = maxObject; this.maxTotal = maxTotal;
+        temporary = root.resolve("pending");
+        catalog = root.resolve("buckets.bin");
+        versions = root.resolve("versions");
+        this.maxObject = maxObject;
+        this.maxTotal = maxTotal;
         Arrays.setAll(locks, i -> new Object());
         Files.createDirectories(root);
         FileChannel channel = FileChannel.open(root.resolve(".process.lock"), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
@@ -45,7 +65,9 @@ final class DiskStore implements ObjectStorage {
             try { acquired = channel.tryLock(); }
             catch (OverlappingFileLockException e) { throw new IOException("Data directory is already in use", e); }
             if (acquired == null) throw new IOException("Data directory is already in use");
-            Files.createDirectories(objects); Files.createDirectories(temporary);
+            Files.createDirectories(objects);
+            Files.createDirectories(temporary);
+            Files.createDirectories(versions);
             syncDirectory(root);
             try (var paths = Files.list(temporary)) {
                 for (Path p : paths.toList()) if (p.getFileName().toString().endsWith(".part")) Files.delete(p);
@@ -67,6 +89,37 @@ final class DiskStore implements ObjectStorage {
                     used = Math.addExact(used, meta.length());
                 }
             }
+            if (Files.exists(catalog)) {
+                try (DataInputStream input = new DataInputStream(Files.newInputStream(catalog))) {
+                    int magic = input.readInt();
+                    if (magic != BUCKET_MAGIC && magic != BUCKET_MAGIC_V2 && magic != BUCKET_MAGIC_V3)
+                        throw new IOException("Invalid bucket catalog");
+                    int count = input.readInt();
+                    if (count < 0 || count > 1000) throw new IOException("Invalid bucket catalog");
+                    for (int i = 0; i < count; i++) {
+                        String name = input.readUTF();
+                        long created = input.readLong();
+                        VersioningState state = VersioningState.NEVER;
+                        if (magic == BUCKET_MAGIC_V2 || magic == BUCKET_MAGIC_V3) {
+                            int ordinal = input.readUnsignedByte();
+                            if (ordinal >= VersioningState.values().length)
+                                throw new IOException("Invalid bucket versioning state");
+                            state = VersioningState.values()[ordinal];
+                        }
+                        Map<String, String> acl = Map.of();
+                        if (magic == BUCKET_MAGIC_V3) {
+                            int size = input.readUnsignedShort();
+                            if (size > ACL_AREA) throw new IOException("Invalid bucket ACL");
+                            acl = ObjectAttributes.decode(input.readNBytes(size));
+                        }
+                        if (!validBucket(name) || created < 0 ||
+                            buckets.put(name, new Bucket(name, created, state, acl)) != null)
+                            throw new IOException("Invalid bucket catalog");
+                    }
+                    if (input.read() != -1) throw new IOException("Invalid bucket catalog");
+                }
+            }
+            loadHistories();
             ready = true;
         } finally {
             if (!ready) {
@@ -85,12 +138,251 @@ final class DiskStore implements ObjectStorage {
     Path root() { return root; }
     long maxObject() { return maxObject; }
     long maxTotal() { return maxTotal; }
+    @Override public Limits limits() { return new Limits(maxObject, maxTotal); }
     synchronized long usedBytes() { return used; }
     synchronized int indexedObjects() { return index.size(); }
     synchronized long objectCount() { return objectCount; }
     synchronized long legacyObjects() { return legacyCount; }
 
+    private static boolean validBucket(String name) {
+        return name.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]");
+    }
+
+    @Override public synchronized void ensureBucket(String bucket) throws IOException {
+        if (!buckets.containsKey(bucket)) createBucket(bucket);
+    }
+
+    @Override public synchronized Bucket bucket(String name) {
+        Bucket found = buckets.get(name);
+        if (found == null) throw new StoreException(404, "NoSuchBucket", "Bucket not found");
+        return found;
+    }
+
+    @Override public synchronized List<Bucket> buckets() { return List.copyOf(buckets.values()); }
+
+    @Override public synchronized void createBucket(String name) throws IOException {
+        if (!validBucket(name)) throw new StoreException(400, "InvalidBucketName", "Invalid bucket name");
+        if (buckets.containsKey(name))
+            throw new StoreException(409, "BucketAlreadyOwnedByYou", "Bucket already exists");
+        if (buckets.size() >= 1000) throw new StoreException(400, "TooManyBuckets", "Bucket limit reached");
+        NavigableMap<String, Bucket> next = new TreeMap<>(buckets);
+        next.put(name, new Bucket(name, Instant.now().toEpochMilli()));
+        saveBuckets(next);
+        buckets.clear();
+        buckets.putAll(next);
+    }
+
+    @Override public synchronized void deleteBucket(String name) throws IOException {
+        bucket(name);
+        if (index.values().stream().anyMatch(meta -> name.equals(meta.bucket())))
+            throw new StoreException(409, "BucketNotEmpty", "Bucket contains objects");
+        if (histories.entrySet().stream().anyMatch(entry -> entry.getKey().startsWith(name + "\0") &&
+            !entry.getValue().isEmpty()))
+            throw new StoreException(409, "BucketNotEmpty", "Bucket contains object versions");
+        if (legacyCount > 0) {
+            try (var paths = Files.walk(objects)) {
+                for (Path path : paths.filter(Files::isRegularFile).toList()) {
+                    try (var input = new DataInputStream(Files.newInputStream(path))) {
+                        Metadata metadata = readRecord(input).metadata();
+                        if (metadata.key() == null && name.equals(metadata.bucket()))
+                            throw new StoreException(409, "BucketNotEmpty", "Bucket contains legacy objects");
+                    }
+                }
+            }
+        }
+        NavigableMap<String, Bucket> next = new TreeMap<>(buckets);
+        next.remove(name);
+        saveBuckets(next);
+        buckets.clear();
+        buckets.putAll(next);
+        for (String key : new ArrayList<>(histories.keySet())) {
+            if (!key.startsWith(name + "\0") || !histories.get(key).isEmpty()) continue;
+            String objectKey = key.substring(name.length() + 1);
+            Path directory = historyDirectory(name, objectKey);
+            Files.deleteIfExists(directory.resolve("manifest"));
+            syncDirectory(directory);
+            histories.remove(key);
+        }
+    }
+
+    private void saveBuckets(NavigableMap<String, Bucket> next) throws IOException {
+        Path pending = Files.createTempFile(temporary, "buckets-", ".part");
+        try {
+            try (DataOutputStream output = new DataOutputStream(Files.newOutputStream(pending))) {
+                output.writeInt(BUCKET_MAGIC_V3);
+                output.writeInt(next.size());
+                for (Bucket entry : next.values()) {
+                    output.writeUTF(entry.name());
+                    output.writeLong(entry.created());
+                    output.writeByte(entry.versioning().ordinal());
+                    byte[] acl = ObjectAttributes.encode(entry.acl(), ACL_AREA);
+                    output.writeShort(acl.length);
+                    output.write(acl);
+                }
+            }
+            try (FileChannel channel = FileChannel.open(pending, StandardOpenOption.WRITE)) { channel.force(true); }
+            Files.move(pending, catalog, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+            syncDirectory(root);
+        } finally { Files.deleteIfExists(pending); }
+    }
+
+    @Override public synchronized void setVersioning(String name, VersioningState state) throws IOException {
+        if (state == VersioningState.NEVER)
+            throw new StoreException(400, "InvalidArgument", "Versioning cannot be disabled after it is enabled");
+        Bucket old = bucket(name);
+        if (old.versioning() == VersioningState.NEVER && state == VersioningState.SUSPENDED)
+            throw new StoreException(400, "InvalidArgument", "Enable versioning before suspending it");
+        NavigableMap<String, Bucket> next = new TreeMap<>(buckets);
+        next.put(name, new Bucket(name, old.created(), state, old.acl()));
+        saveBuckets(next);
+        buckets.clear();
+        buckets.putAll(next);
+    }
+
+    @Override public synchronized void setBucketAcl(String name, Map<String, String> acl) throws IOException {
+        Bucket old = bucket(name);
+        NavigableMap<String, Bucket> next = new TreeMap<>(buckets);
+        next.put(name, new Bucket(name, old.created(), old.versioning(), Map.copyOf(acl)));
+        saveBuckets(next);
+        buckets.clear();
+        buckets.putAll(next);
+    }
+
     private static String indexKey(String bucket, String key) { return bucket + "\0" + key; }
+    private Path historyDirectory(String bucket, String key) {
+        String id = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(StandardCharsets.UTF_8)));
+        return versions.resolve(id.substring(0, 2)).resolve(id);
+    }
+    private Path versionPath(String bucket, String key, String storageId) {
+        return storageId.equals("legacy") ? objectPath(bucket, key) :
+            historyDirectory(bucket, key).resolve(storageId);
+    }
+    private static Metadata withVersion(Metadata old, String id, String bucket, String key) {
+        return new Metadata(old.length(), old.modified(), old.etag(), old.sha256(), bucket, key,
+            old.contentType(), old.userMetadata(), old.tags(), id, old.checksums(), old.acl());
+    }
+    private List<VersionRecord> history(String bucket, String key) throws IOException {
+        List<VersionRecord> found = histories.get(indexKey(bucket, key));
+        if (found != null) return found;
+        Metadata old = index.get(indexKey(bucket, key));
+        if (old == null && Files.isRegularFile(objectPath(bucket, key))) {
+            try (DataInputStream input = new DataInputStream(Files.newInputStream(objectPath(bucket, key)))) {
+                old = readRecord(input).metadata();
+            }
+        }
+        return old == null ? List.of() : List.of(new VersionRecord("null", "legacy", false, old.modified()));
+    }
+    private void saveHistory(String bucket, String key, List<VersionRecord> entries) throws IOException {
+        Path directory = historyDirectory(bucket, key);
+        if (!Files.isDirectory(directory)) {
+            Files.createDirectories(directory);
+            syncDirectory(directory.getParent());
+        }
+        Path pending = Files.createTempFile(temporary, "history-", ".part");
+        try {
+            try (DataOutputStream output = new DataOutputStream(Files.newOutputStream(pending))) {
+                output.writeInt(VERSION_MAGIC);
+                output.writeUTF(bucket);
+                output.writeUTF(key);
+                output.writeInt(entries.size());
+                for (VersionRecord entry : entries) {
+                    output.writeUTF(entry.id());
+                    output.writeUTF(entry.storageId());
+                    output.writeBoolean(entry.marker());
+                    output.writeLong(entry.modified());
+                }
+            }
+            try (FileChannel channel = FileChannel.open(pending, StandardOpenOption.WRITE)) {
+                channel.force(true);
+            }
+            Files.move(pending, directory.resolve("manifest"), StandardCopyOption.ATOMIC_MOVE,
+                StandardCopyOption.REPLACE_EXISTING);
+            syncDirectory(directory);
+        } finally { Files.deleteIfExists(pending); }
+        histories.put(indexKey(bucket, key), List.copyOf(entries));
+    }
+    private void loadHistories() throws IOException {
+        Set<Path> referenced = new HashSet<>();
+        try (var paths = Files.walk(versions)) {
+            for (Path manifest : paths.filter(p -> p.getFileName().toString().equals("manifest")).toList()) {
+                String bucket;
+                String key;
+                List<VersionRecord> entries = new ArrayList<>();
+                try (DataInputStream input = new DataInputStream(Files.newInputStream(manifest))) {
+                    if (input.readInt() != VERSION_MAGIC) throw new IOException("Invalid version manifest: " + manifest);
+                    bucket = input.readUTF();
+                    key = input.readUTF();
+                    int count = input.readInt();
+                    if (count < 0 || count > 1_000_000 || !manifest.getParent().equals(historyDirectory(bucket, key)))
+                        throw new IOException("Invalid version manifest: " + manifest);
+                    Set<String> ids = new HashSet<>();
+                    for (int i = 0; i < count; i++) {
+                        String id = input.readUTF();
+                        String storageId = input.readUTF();
+                        boolean marker = input.readBoolean();
+                        long modified = input.readLong();
+                        if ((!id.equals("null") && !id.matches("[0-9a-f-]{36}")) ||
+                            (!marker && !storageId.equals("legacy") && !storageId.matches("[0-9a-f-]{36}")) ||
+                            (marker && !storageId.isEmpty()) || !ids.add(id) || modified < 0)
+                            throw new IOException("Invalid version manifest entry: " + manifest);
+                        entries.add(new VersionRecord(id, storageId, marker, modified));
+                        if (!marker) {
+                            Path file = versionPath(bucket, key, storageId);
+                            if (!Files.isRegularFile(file)) throw new IOException("Missing object version: " + file);
+                            referenced.add(file);
+                            if (!storageId.equals("legacy")) {
+                                Record record;
+                                try (DataInputStream data = new DataInputStream(Files.newInputStream(file))) {
+                                    record = readRecord(data);
+                                }
+                                Metadata meta = record.metadata();
+                                if (!bucket.equals(meta.bucket()) || !key.equals(meta.key()) ||
+                                    Files.size(file) - record.headerLength() != meta.length())
+                                    throw new IOException("Invalid object version: " + file);
+                                objectCount++;
+                                used = Math.addExact(used, meta.length());
+                            }
+                        }
+                    }
+                    if (input.read() != -1 || histories.put(indexKey(bucket, key), List.copyOf(entries)) != null)
+                        throw new IOException("Invalid version manifest: " + manifest);
+                }
+                if (entries.isEmpty() || entries.getFirst().marker()) index.remove(indexKey(bucket, key));
+                else {
+                    VersionRecord current = entries.getFirst();
+                    Path file = versionPath(bucket, key, current.storageId());
+                    try (DataInputStream input = new DataInputStream(Files.newInputStream(file))) {
+                        index.put(indexKey(bucket, key),
+                            withVersion(readRecord(input).metadata(), current.id(), bucket, key));
+                    }
+                }
+            }
+        }
+        try (var paths = Files.walk(versions)) {
+            for (Path file : paths.filter(Files::isRegularFile).toList()) {
+                if (!file.getFileName().toString().equals("manifest") && !referenced.contains(file)) {
+                    Files.delete(file);
+                    syncDirectory(file.getParent());
+                }
+            }
+        }
+        for (var entry : histories.entrySet()) {
+            String[] parts = entry.getKey().split("\0", 2);
+            if (entry.getValue().stream().anyMatch(version -> version.storageId().equals("legacy") && !version.marker()))
+                continue;
+            Path orphan = objectPath(parts[0], parts[1]);
+            if (Files.isRegularFile(orphan)) {
+                try (DataInputStream input = new DataInputStream(Files.newInputStream(orphan))) {
+                    Metadata old = readRecord(input).metadata();
+                    used -= old.length();
+                    if (old.key() == null) legacyCount--;
+                }
+                Files.delete(orphan);
+                objectCount--;
+                syncDirectory(orphan.getParent());
+            }
+        }
+    }
     private Path objectPath(String bucket, String key) {
         String id = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(StandardCharsets.UTF_8)));
         return objects.resolve(id.substring(0, 2)).resolve(id);
@@ -111,47 +403,97 @@ final class DiskStore implements ObjectStorage {
     private Object lock(Path p) { return locks[(p.hashCode() & 0x7fffffff) % locks.length]; }
 
     public Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
-                 String checksum, boolean createOnly, String contentType) throws IOException {
+                 String checksum, boolean createOnly, String contentType,
+                 Map<String, String> userMetadata, Map<String, String> tags,
+                 java.util.function.Supplier<Map<String, String>> checksums,
+                 Map<String, String> acl) throws IOException {
         if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required");
         if (length > maxObject) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit");
         byte[] bucketBytes = bucket.getBytes(StandardCharsets.UTF_8);
         byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
         byte[] typeBytes = contentType.getBytes(StandardCharsets.UTF_8);
-        if (bucketBytes.length > 63 || keyBytes.length > 1024 || typeBytes.length > 255)
+        byte[] metadataBytes = ObjectAttributes.encode(userMetadata, 4096);
-            throw new StoreException(400, "InvalidArgument", "Object metadata is too long");
+        byte[] tagBytes = ObjectAttributes.encode(tags, 8192);
-        int headerLength = HEADER_V2 + bucketBytes.length + keyBytes.length + typeBytes.length;
+        validateMetadataLengths(bucketBytes, keyBytes, typeBytes);
         Path destination = object(bucket, key), pending = Files.createTempFile(temporary, "upload-", ".part");
         try {
+            Metadata metadata = stagePut(pending, input, length, expectedHash, checksum,
+                bucket, key, contentType, bucketBytes, keyBytes, typeBytes,
+                metadataBytes, tagBytes, userMetadata, tags, checksums, acl);
+            return installPending(destination, pending, metadata, createOnly);
+        } finally { Files.deleteIfExists(pending); }
+    }
+
+    private static void validateMetadataLengths(byte[] bucketBytes, byte[] keyBytes, byte[] typeBytes) {
+        if (bucketBytes.length > 63 || keyBytes.length > 1024 || typeBytes.length > 255)
+            throw new StoreException(400, "InvalidArgument", "Object metadata is too long");
+    }
+
+    private Metadata stagePut(Path pending, InputStream input, long length, String expectedHash, String checksum,
+                              String bucket, String key, String contentType,
+                              byte[] bucketBytes, byte[] keyBytes, byte[] typeBytes,
+                              byte[] metadataBytes, byte[] tagBytes,
+                              Map<String, String> userMetadata, Map<String, String> tags,
+                              java.util.function.Supplier<Map<String, String>> checksums,
+                              Map<String, String> acl) throws IOException {
+        byte[] aclBytes = ObjectAttributes.encode(acl, ACL_AREA);
+        int headerLength = HEADER_V5 + aclBytes.length +
+            bucketBytes.length + keyBytes.length + typeBytes.length +
+            metadataBytes.length + tagBytes.length;
         MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
+        Crc64Nvme crc64 = new Crc64Nvme();
         long count = 0;
         try (OutputStream out = Files.newOutputStream(pending)) {
             out.write(new byte[headerLength]);
-                byte[] buffer = new byte[65536]; int n;
+            byte[] buffer = new byte[65536];
+            int n;
             while ((n = input.read(buffer)) != -1) {
                 count += n;
                 if (count > length || count > maxObject)
                     throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size");
-                    sha.update(buffer, 0, n); md5.update(buffer, 0, n); out.write(buffer, 0, n);
+                sha.update(buffer, 0, n);
+                md5.update(buffer, 0, n);
+                crc64.update(buffer, 0, n);
+                out.write(buffer, 0, n);
             }
         }
         if (count != length) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length");
         byte[] hash = sha.digest(), etag = md5.digest();
-            if (!MessageDigest.isEqual(hash, HexFormat.of().parseHex(expectedHash)))
+        if (expectedHash != null && !MessageDigest.isEqual(hash, HexFormat.of().parseHex(expectedHash)))
             throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch");
         if (checksum != null && !Base64.getEncoder().encodeToString(hash).equals(checksum))
             throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch");
+        Map<String, String> suppliedChecksums = checksums.get();
+        Map<String, String> storedChecksums = suppliedChecksums.isEmpty() ?
+            Map.of("x-amz-checksum-crc64nvme", crc64.encoded()) : Map.copyOf(suppliedChecksums);
+        byte[] checksumBytes = ObjectAttributes.encode(storedChecksums, CHECKSUM_AREA);
         long modified = Instant.now().toEpochMilli();
-            ByteBuffer header = ByteBuffer.allocate(headerLength).putLong(MAGIC_V2).putLong(count)
+        ByteBuffer header = ByteBuffer.allocate(headerLength).putLong(MAGIC_V5).putLong(count)
             .putLong(modified).put(etag).put(hash).putShort((short) bucketBytes.length)
             .putShort((short) keyBytes.length).putShort((short) typeBytes.length)
-                .put(bucketBytes).put(keyBytes).put(typeBytes);
+            .putShort((short) metadataBytes.length).putShort((short) tagBytes.length)
+            .putShort((short) checksumBytes.length);
+        header.position(header.position() + CHECKSUM_AREA - checksumBytes.length);
+        header.put(checksumBytes).putShort((short) aclBytes.length);
+        header.put(aclBytes).put(bucketBytes).put(keyBytes).put(typeBytes).put(metadataBytes).put(tagBytes);
         header.flip();
         try (FileChannel file = FileChannel.open(pending, StandardOpenOption.WRITE)) {
             while (header.hasRemaining()) file.write(header, header.position());
             file.force(true);
         }
-            Metadata metadata = new Metadata(count, modified, SigV4.hex(etag), hash, bucket, key, contentType);
+        return new Metadata(count, modified, SigV4.hex(etag), hash, bucket, key, contentType,
+            Map.copyOf(userMetadata), Map.copyOf(tags), null, storedChecksums, Map.copyOf(acl));
+    }
+
+    private Metadata installPending(Path destination, Path pending, Metadata metadata,
+                                    boolean createOnly) throws IOException {
         synchronized (lock(destination)) {
+            synchronized (this) {
+                Bucket configured = buckets.get(metadata.bucket());
+                if (configured != null && configured.versioning() != VersioningState.NEVER)
+                    return installVersionedPending(destination, pending, metadata, createOnly,
+                        configured.versioning());
+            }
             long previous = 0;
             boolean existed = Files.exists(destination);
             boolean legacy = false;
@@ -164,35 +506,123 @@ final class DiskStore implements ObjectStorage {
                 }
             }
             synchronized (this) {
-                    if (used - previous + count > maxTotal)
+                if (Files.exists(catalog) && !buckets.containsKey(metadata.bucket()))
+                    throw new StoreException(404, "NoSuchBucket", "Bucket not found");
+                if (used - previous + metadata.length() > maxTotal)
                     throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached");
                 Files.move(pending, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
-                    used = used - previous + count;
+                used = used - previous + metadata.length();
                 if (!existed) objectCount++;
                 if (legacy) legacyCount--;
-                    index.put(indexKey(bucket, key), metadata);
+                index.put(indexKey(metadata.bucket(), metadata.key()), metadata);
                 syncDirectory(destination.getParent());
             }
-            }
             return metadata;
-        } finally { Files.deleteIfExists(pending); }
+        }
+    }
+
+    private Metadata installVersionedPending(Path destination, Path pending, Metadata metadata,
+                                             boolean createOnly, VersioningState state) throws IOException {
+        String bucket = metadata.bucket();
+        String key = metadata.key();
+        if (createOnly && index.containsKey(indexKey(bucket, key)))
+            throw new StoreException(412, "PreconditionFailed", "Object already exists");
+        List<VersionRecord> old = history(bucket, key);
+        String id = state == VersioningState.ENABLED ? UUID.randomUUID().toString() : "null";
+        String storageId = UUID.randomUUID().toString();
+        VersionRecord discarded = null;
+        long replaced = 0;
+        if (id.equals("null")) {
+            for (VersionRecord entry : old) {
+                if (!entry.id().equals("null") || entry.marker()) continue;
+                discarded = entry;
+                try (DataInputStream input = new DataInputStream(Files.newInputStream(
+                    versionPath(bucket, key, entry.storageId())))) {
+                    replaced = readRecord(input).metadata().length();
+                }
+            }
+        }
+        if (maxTotal - (used - replaced) < metadata.length())
+            throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached");
+        Path target = versionPath(bucket, key, storageId);
+        Files.createDirectories(target.getParent());
+        syncDirectory(target.getParent().getParent());
+        Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+        syncDirectory(target.getParent());
+        List<VersionRecord> next = new ArrayList<>();
+        next.add(new VersionRecord(id, storageId, false, metadata.modified()));
+        for (VersionRecord entry : old) if (!entry.id().equals(id)) next.add(entry);
+        saveHistory(bucket, key, next);
+        used += metadata.length();
+        objectCount++;
+        Metadata current = withVersion(metadata, id, bucket, key);
+        index.put(indexKey(bucket, key), current);
+        if (discarded != null) removeStoredVersion(bucket, key, discarded, replaced);
+        return current;
+    }
+
+    private void removeStoredVersion(String bucket, String key, VersionRecord entry, long length) throws IOException {
+        Path file = versionPath(bucket, key, entry.storageId());
+        if (entry.storageId().equals("legacy")) {
+            try (DataInputStream input = new DataInputStream(Files.newInputStream(file))) {
+                if (readRecord(input).metadata().key() == null) legacyCount--;
+            }
+        }
+        Files.deleteIfExists(file);
+        syncDirectory(file.getParent());
+        used -= length;
+        objectCount--;
     }
 
     public OpenObject open(String bucket, String key) throws IOException {
+        return open(bucket, key, null);
+    }
+
+    @Override public OpenObject open(String bucket, String key, String versionId) throws IOException {
         Path destination = object(bucket, key);
         synchronized (lock(destination)) {
+            VersionRecord selected;
+            synchronized (this) {
+                List<VersionRecord> entries = history(bucket, key);
+                selected = versionId == null ? (entries.isEmpty() ? null : entries.getFirst()) :
+                    entries.stream().filter(entry -> entry.id().equals(versionId)).findFirst().orElse(null);
+            }
+            if (selected == null)
+                throw new StoreException(404, versionId == null ? "NoSuchKey" : "NoSuchVersion",
+                    "Object version not found");
+            if (selected.marker())
+                throw StoreException.deletedVersion(selected.id(), selected.modified(), versionId != null);
             final DataInputStream input;
-            try { input = new DataInputStream(Files.newInputStream(destination)); }
+            try { input = new DataInputStream(Files.newInputStream(
+                versionPath(bucket, key, selected.storageId()))); }
             catch (NoSuchFileException e) { throw new StoreException(404, "NoSuchKey", "Object not found"); }
-            try { return new OpenObject(readRecord(input).metadata(), input); }
+            try {
-            catch (IOException e) { input.close(); throw e; }
+                Bucket configured = buckets.get(bucket);
+                String exposedId = histories.containsKey(indexKey(bucket, key)) ||
+                    configured != null && configured.versioning() != VersioningState.NEVER ? selected.id() : null;
+                return new OpenObject(withVersion(readRecord(input).metadata(), exposedId, bucket, key), input);
+            }
+            catch (IOException e) {
+                input.close();
+                throw e;
+            }
         }
     }
 
     public void delete(String bucket, String key) throws IOException {
+        delete(bucket, key, null);
+    }
+
+    @Override public DeleteResult delete(String bucket, String key, String versionId) throws IOException {
         Path destination = object(bucket, key);
         synchronized (lock(destination)) {
-            if (!Files.exists(destination)) return;
+            synchronized (this) {
+                Bucket configured = buckets.get(bucket);
+                VersioningState state = configured == null ? VersioningState.NEVER : configured.versioning();
+                if (state != VersioningState.NEVER || versionId != null)
+                    return deleteVersioned(bucket, key, versionId, state);
+            }
+            if (!Files.exists(destination)) return new DeleteResult(null, false);
             long length;
             boolean legacy;
             try (var input = new DataInputStream(Files.newInputStream(destination))) {
@@ -208,9 +638,146 @@ final class DiskStore implements ObjectStorage {
                 index.remove(indexKey(bucket, key));
                 syncDirectory(destination.getParent());
             }
+            return new DeleteResult(null, false);
         }
     }
 
+    private DeleteResult deleteVersioned(String bucket, String key, String versionId,
+                                         VersioningState state) throws IOException {
+        List<VersionRecord> old = history(bucket, key);
+        List<VersionRecord> next = new ArrayList<>();
+        VersionRecord removed = null;
+        if (versionId != null) {
+            for (VersionRecord entry : old) {
+                if (entry.id().equals(versionId)) removed = entry;
+                else next.add(entry);
+            }
+            if (removed == null)
+                throw new StoreException(404, "NoSuchVersion", "Object version not found");
+        } else {
+            String id = state == VersioningState.ENABLED ? UUID.randomUUID().toString() : "null";
+            next.add(new VersionRecord(id, "", true, Instant.now().toEpochMilli()));
+            for (VersionRecord entry : old) {
+                if (state == VersioningState.SUSPENDED && entry.id().equals("null")) removed = entry;
+                else next.add(entry);
+            }
+        }
+        saveHistory(bucket, key, next);
+        updateCurrentIndex(bucket, key, next);
+        if (removed != null && !removed.marker()) {
+            Path file = versionPath(bucket, key, removed.storageId());
+            long length;
+            try (DataInputStream input = new DataInputStream(Files.newInputStream(file))) {
+                length = readRecord(input).metadata().length();
+            }
+            removeStoredVersion(bucket, key, removed, length);
+        }
+        if (versionId != null) return new DeleteResult(versionId, removed.marker());
+        return new DeleteResult(next.getFirst().id(), true);
+    }
+
+    private void updateCurrentIndex(String bucket, String key, List<VersionRecord> entries) throws IOException {
+        if (entries.isEmpty() || entries.getFirst().marker()) {
+            index.remove(indexKey(bucket, key));
+            return;
+        }
+        VersionRecord latest = entries.getFirst();
+        try (DataInputStream input = new DataInputStream(Files.newInputStream(
+            versionPath(bucket, key, latest.storageId())))) {
+            index.put(indexKey(bucket, key),
+                withVersion(readRecord(input).metadata(), latest.id(), bucket, key));
+        }
+    }
+
+    @Override public Map<String, String> tags(String bucket, String key) throws IOException {
+        return tags(bucket, key, null);
+    }
+
+    @Override public Map<String, String> tags(String bucket, String key, String versionId) throws IOException {
+        try (OpenObject object = open(bucket, key, versionId)) { return object.metadata().tags(); }
+    }
+
+    @Override public void setTags(String bucket, String key, Map<String, String> tags) throws IOException {
+        setTags(bucket, key, null, tags);
+    }
+
+    @Override public void setTags(String bucket, String key, String versionId,
+                                  Map<String, String> tags) throws IOException {
+        rewriteAttributes(bucket, key, versionId, tags, null);
+    }
+
+    @Override public void setObjectAcl(String bucket, String key, String versionId,
+                                       Map<String, String> acl) throws IOException {
+        rewriteAttributes(bucket, key, versionId, null, acl);
+    }
+
+    private void rewriteAttributes(String bucket, String key, String versionId,
+                                   Map<String, String> tags, Map<String, String> acl) throws IOException {
+        Path destination = object(bucket, key);
+        synchronized (lock(destination)) {
+            VersionRecord selected;
+            boolean current;
+            synchronized (this) {
+                List<VersionRecord> entries = history(bucket, key);
+                selected = versionId == null ? (entries.isEmpty() ? null : entries.getFirst()) :
+                    entries.stream().filter(entry -> entry.id().equals(versionId)).findFirst().orElse(null);
+                current = selected != null && !entries.isEmpty() && selected == entries.getFirst();
+            }
+            if (selected == null || selected.marker())
+                throw new StoreException(404, versionId == null ? "NoSuchKey" : "NoSuchVersion",
+                    "Object version not found");
+            destination = versionPath(bucket, key, selected.storageId());
+            Path pending = Files.createTempFile(temporary, "tags-", ".part");
+            try {
+                try (DataInputStream input = new DataInputStream(Files.newInputStream(destination));
+                     OutputStream output = Files.newOutputStream(pending)) {
+                    Metadata old = readRecord(input).metadata();
+                    if (old.key() == null) throw new StoreException(501, "NotImplemented", "Legacy object tags are unsupported");
+                    Metadata updated = new Metadata(old.length(), old.modified(), old.etag(), old.sha256(),
+                        bucket, key, old.contentType(), old.userMetadata(),
+                        tags == null ? old.tags() : Map.copyOf(tags),
+                        old.versionId(), old.checksums(),
+                        acl == null ? old.acl() : Map.copyOf(acl));
+                    output.write(recordHeader(updated));
+                    if (input.transferTo(output) != old.length()) throw new IOException("Object length changed during tag update");
+                }
+                try (FileChannel channel = FileChannel.open(pending, StandardOpenOption.WRITE)) { channel.force(true); }
+                synchronized (this) {
+                    Files.move(pending, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+                    if (current) {
+                        Metadata old = index.get(indexKey(bucket, key));
+                        index.put(indexKey(bucket, key), new Metadata(old.length(), old.modified(), old.etag(),
+                            old.sha256(), bucket, key, old.contentType(), old.userMetadata(),
+                            tags == null ? old.tags() : Map.copyOf(tags),
+                            old.versionId(), old.checksums(),
+                            acl == null ? old.acl() : Map.copyOf(acl)));
+                    }
+                    syncDirectory(destination.getParent());
+                }
+            } finally { Files.deleteIfExists(pending); }
+        }
+    }
+
+    private static byte[] recordHeader(Metadata metadata) {
+        byte[] bucket = metadata.bucket().getBytes(StandardCharsets.UTF_8);
+        byte[] key = metadata.key().getBytes(StandardCharsets.UTF_8);
+        byte[] type = metadata.contentType().getBytes(StandardCharsets.UTF_8);
+        byte[] custom = ObjectAttributes.encode(metadata.userMetadata(), 4096);
+        byte[] tags = ObjectAttributes.encode(metadata.tags(), 8192);
+        byte[] checksums = ObjectAttributes.encode(metadata.checksums(), CHECKSUM_AREA);
+        byte[] acl = ObjectAttributes.encode(metadata.acl(), ACL_AREA);
+        ByteBuffer header = ByteBuffer.allocate(HEADER_V5 + acl.length +
+            bucket.length + key.length + type.length + custom.length + tags.length)
+            .putLong(MAGIC_V5).putLong(metadata.length()).putLong(metadata.modified())
+            .put(HexFormat.of().parseHex(metadata.etag())).put(metadata.sha256())
+            .putShort((short) bucket.length).putShort((short) key.length).putShort((short) type.length)
+            .putShort((short) custom.length).putShort((short) tags.length)
+            .putShort((short) checksums.length);
+        header.position(header.position() + CHECKSUM_AREA - checksums.length);
+        header.put(checksums).putShort((short) acl.length);
+        return header.put(acl).put(bucket).put(key).put(type).put(custom).put(tags).array();
+    }
+
     public synchronized ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) {
         List<ListedObject> entries = new ArrayList<>();
         List<String> prefixes = new ArrayList<>();
@@ -227,28 +794,111 @@ final class DiskStore implements ObjectStorage {
                 int at = key.indexOf(delimiter, prefix.length());
                 if (at >= 0) group = key.substring(0, at + delimiter.length());
             }
-            if (group != null && group.equals(activePrefix)) { lastKey = key; continue; }
+            if (group != null && group.equals(activePrefix)) {
-            if (entries.size() + prefixes.size() >= maxKeys) { truncated = true; break; }
+                lastKey = key;
-            if (group != null) { prefixes.add(group); activePrefix = group; }
+                continue;
-            else { entries.add(new ListedObject(key, meta)); activePrefix = null; }
+            }
+            if (entries.size() + prefixes.size() >= maxKeys) {
+                truncated = true;
+                break;
+            }
+            if (group != null) {
+                prefixes.add(group);
+                activePrefix = group;
+            } else {
+                entries.add(new ListedObject(key, meta));
+                activePrefix = null;
+            }
             lastKey = key;
         }
         return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated);
     }
 
+    @Override public synchronized VersionPage listVersions(String bucket, String prefix,
+                                                             String keyMarker, String versionMarker,
+                                                             int maxKeys) throws IOException {
+        bucket(bucket);
+        if (versionMarker != null && keyMarker == null)
+            throw new StoreException(400, "InvalidArgument", "Version marker requires a key marker");
+        if (maxKeys == 0) return new VersionPage(List.of(), null, null, false);
+        NavigableSet<String> keys = new TreeSet<>();
+        for (String name : histories.keySet()) if (name.startsWith(bucket + "\0")) keys.add(name.substring(bucket.length() + 1));
+        for (Metadata meta : index.values()) if (bucket.equals(meta.bucket())) keys.add(meta.key());
+        List<VersionEntry> page = new ArrayList<>();
+        String nextKey = null;
+        String nextVersion = null;
+        boolean truncated = false;
+        for (String key : keys) {
+            if (!key.startsWith(prefix) || keyMarker != null && key.compareTo(keyMarker) < 0) continue;
+            List<VersionRecord> entries = history(bucket, key);
+            boolean pastMarker = keyMarker == null || !key.equals(keyMarker) || versionMarker == null;
+            for (int i = 0; i < entries.size(); i++) {
+                VersionRecord entry = entries.get(i);
+                if (keyMarker != null && key.equals(keyMarker)) {
+                    if (versionMarker == null) continue;
+                    if (!pastMarker) {
+                        if (entry.id().equals(versionMarker)) pastMarker = true;
+                        continue;
+                    }
+                }
+                if (page.size() == maxKeys) {
+                    truncated = true;
+                    break;
+                }
+                Metadata meta = null;
+                if (!entry.marker()) {
+                    try (DataInputStream input = new DataInputStream(Files.newInputStream(
+                        versionPath(bucket, key, entry.storageId())))) {
+                        meta = withVersion(readRecord(input).metadata(), entry.id(), bucket, key);
+                    }
+                }
+                page.add(new VersionEntry(key, entry.id(), entry.modified(), entry.marker(), i == 0, meta));
+                nextKey = key;
+                nextVersion = entry.id();
+            }
+            if (truncated) break;
+        }
+        return new VersionPage(page, truncated ? nextKey : null, truncated ? nextVersion : null, truncated);
+    }
+
     static Record readRecord(DataInputStream in) throws IOException {
         long magic = in.readLong();
-        if (magic != MAGIC_V1 && magic != MAGIC_V2) throw new IOException("Invalid object record");
+        if (magic != MAGIC_V1 && magic != MAGIC_V2 && magic != MAGIC_V3 && magic != MAGIC_V4 &&
+            magic != MAGIC_V5)
+            throw new IOException("Invalid object record");
         long length = in.readLong(), modified = in.readLong();
         byte[] md5 = new byte[16], sha = new byte[32];
-        in.readFully(md5); in.readFully(sha);
+        in.readFully(md5);
+        in.readFully(sha);
         if (length < 0) throw new IOException("Invalid object record length");
         if (magic == MAGIC_V1)
             return new Record(new Metadata(length, modified, SigV4.hex(md5), sha,
                 null, null, "application/octet-stream"), HEADER_V1);
         int bucketLength = in.readUnsignedShort(), keyLength = in.readUnsignedShort(), typeLength = in.readUnsignedShort();
+        int metadataLength = magic == MAGIC_V3 || magic == MAGIC_V4 || magic == MAGIC_V5 ? in.readUnsignedShort() : 0;
+        int tagsLength = magic == MAGIC_V3 || magic == MAGIC_V4 || magic == MAGIC_V5 ? in.readUnsignedShort() : 0;
+        int checksumLength = magic == MAGIC_V4 || magic == MAGIC_V5 ? in.readUnsignedShort() : 0;
         if (bucketLength < 1 || bucketLength > 63 || keyLength < 1 || keyLength > 1024 || typeLength < 1 || typeLength > 255)
             throw new IOException("Invalid object record metadata");
+        if (metadataLength > 4096 || tagsLength > 8192 || checksumLength > CHECKSUM_AREA)
+            throw new IOException("Invalid object attributes");
+        Map<String, String> checksums = Map.of();
+        if (magic == MAGIC_V4 || magic == MAGIC_V5) {
+            byte[] area = in.readNBytes(CHECKSUM_AREA);
+            if (area.length != CHECKSUM_AREA) throw new IOException("Truncated checksum attributes");
+            checksums = ObjectAttributes.decode(Arrays.copyOfRange(area,
+                CHECKSUM_AREA - checksumLength, CHECKSUM_AREA));
+        }
+        Map<String, String> acl = Map.of();
+        int aclLength = 0;
+        if (magic == MAGIC_V5) {
+            int size = in.readUnsignedShort();
+            if (size > ACL_AREA) throw new IOException("Invalid object ACL");
+            byte[] bytes = in.readNBytes(size);
+            if (bytes.length != size) throw new IOException("Truncated object ACL");
+            acl = ObjectAttributes.decode(bytes);
+            aclLength = size;
+        }
         String bucket = utf8(in.readNBytes(bucketLength));
         String key = utf8(in.readNBytes(keyLength));
         String contentType = utf8(in.readNBytes(typeLength));
@@ -256,8 +906,13 @@ final class DiskStore implements ObjectStorage {
             key.getBytes(StandardCharsets.UTF_8).length != keyLength ||
             contentType.getBytes(StandardCharsets.UTF_8).length != typeLength)
             throw new IOException("Invalid object record metadata");
+        Map<String, String> metadata = ObjectAttributes.decode(in.readNBytes(metadataLength));
+        Map<String, String> tags = ObjectAttributes.decode(in.readNBytes(tagsLength));
         return new Record(new Metadata(length, modified, SigV4.hex(md5), sha,
-            bucket, key, contentType), HEADER_V2 + bucketLength + keyLength + typeLength);
+            bucket, key, contentType, metadata, tags, null, checksums, acl),
+            (magic == MAGIC_V5 ? HEADER_V5 + aclLength :
+             magic == MAGIC_V4 ? HEADER_V4 : magic == MAGIC_V3 ? HEADER_V3 : HEADER_V2) +
+                bucketLength + keyLength + typeLength + metadataLength + tagsLength);
     }
     private static String utf8(byte[] bytes) throws IOException {
         return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)

src/cloud/lunarsky/store/EncryptedVolume.java

@@ -0,0 +1,29 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+final class EncryptedVolume {
+    private EncryptedVolume() {}
+
+    static void requireConfigured(Map<String, String> environment) throws IOException {
+        requireConfigured(environment, null);
+    }
+
+    static void requireConfigured(Map<String, String> environment, Path dataDirectory) throws IOException {
+        String marker = environment.getOrDefault("ENCRYPTED_VOLUME_MARKER_FILE", "");
+        String expected = environment.getOrDefault("ENCRYPTED_VOLUME_ID", "");
+        if (marker.isEmpty() && expected.isEmpty()) return;
+        if (!expected.matches("[a-f0-9]{32}") || marker.isEmpty())
+            throw new IOException("Encrypted storage marker configuration is incomplete");
+        Path path = Path.of(marker);
+        if (!path.isAbsolute() || dataDirectory != null &&
+            !path.normalize().equals(dataDirectory.toAbsolutePath().normalize().resolve(".objectstore-encrypted")) ||
+            Files.isSymbolicLink(path) || !Files.isRegularFile(path) ||
+            Files.size(path) > 33 || !Files.readString(path, StandardCharsets.US_ASCII).trim().equals(expected))
+            throw new IOException("Encrypted storage volume is unavailable or does not match its marker");
+    }
+}

src/cloud/lunarsky/store/MultipartStorage.java

@@ -3,15 +3,30 @@ package cloud.lunarsky.store;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Map;
 
 interface MultipartStorage {
     record Part(int number, String etag) {}
+    record PartInfo(int number, long length, String etag, long modified) {}
+    record PartPage(List<PartInfo> parts, int nextMarker, boolean truncated) {}
+    record UploadInfo(String id, String key, long created) {}
 
-    String create(String bucket, String key, String contentType) throws IOException;
+    String create(String bucket, String key, String contentType,
+                  Map<String, String> userMetadata, Map<String, String> tags,
+                  Map<String, String> acl) throws IOException;
+    default String create(String bucket, String key, String contentType,
+                          Map<String, String> userMetadata, Map<String, String> tags) throws IOException {
+        return create(bucket, key, contentType, userMetadata, tags, Map.of());
+    }
+    default String create(String bucket, String key, String contentType) throws IOException {
+        return create(bucket, key, contentType, Map.of(), Map.of());
+    }
     String putPart(String id, String bucket, String key, int number, InputStream input,
                    long length, String expectedHash, String checksum) throws IOException;
     ObjectStorage.Metadata complete(String id, String bucket, String key, List<Part> parts) throws IOException;
     void abort(String id, String bucket, String key) throws IOException;
+    PartPage listParts(String id, String bucket, String key, int marker, int maxParts) throws IOException;
+    List<UploadInfo> listUploads(String bucket, String prefix) throws IOException;
     int activeUploads();
     long stagedBytes();
 }

src/cloud/lunarsky/store/MultipartStore.java

@@ -8,12 +8,16 @@ import cloud.lunarsky.store.MultipartStorage.Part;
 
 final class MultipartStore implements MultipartStorage {
     private static final int MAGIC = 0x4c534d50;
+    private static final int MAGIC_V2 = 0x4c534d51;
+    private static final int MAGIC_V3 = 0x4c534d52;
     private final DiskStore store;
     private final Path root;
     private long staged;
     private int active;
 
-    private record Upload(String bucket, String key, String contentType) {}
+    private record Upload(String bucket, String key, String contentType,
+                          Map<String, String> userMetadata, Map<String, String> tags,
+                          Map<String, String> acl) {}
 
     MultipartStore(DiskStore store) throws IOException {
         this.store = store;
@@ -42,17 +46,29 @@ final class MultipartStore implements MultipartStorage {
         if (staged > store.maxTotal()) throw new IOException("Multipart staging limit exceeded");
     }
 
-    public synchronized String create(String bucket, String key, String contentType) throws IOException {
+    public synchronized String create(String bucket, String key, String contentType,
+                                      Map<String, String> userMetadata, Map<String, String> tags,
+                                      Map<String, String> acl) throws IOException {
+        if (Files.exists(store.root().resolve("buckets.bin"))) store.bucket(bucket);
         if (active >= 32) throw new StoreException(503, "SlowDown", "Too many active uploads");
         String id = UUID.randomUUID().toString();
         Path pending = root.resolve(".creating-" + id), dir = root.resolve(id);
         Files.createDirectory(pending);
         try {
             try (var output = new DataOutputStream(Files.newOutputStream(pending.resolve("manifest"), StandardOpenOption.CREATE_NEW))) {
-                output.writeInt(MAGIC);
+                output.writeInt(MAGIC_V3);
                 output.writeUTF(bucket);
                 output.writeUTF(key);
                 output.writeUTF(contentType);
+                byte[] custom = ObjectAttributes.encode(userMetadata, 4096);
+                byte[] encodedTags = ObjectAttributes.encode(tags, 8192);
+                output.writeShort(custom.length);
+                output.write(custom);
+                output.writeShort(encodedTags.length);
+                output.write(encodedTags);
+                byte[] encodedAcl = ObjectAttributes.encode(acl, 2048);
+                output.writeShort(encodedAcl.length);
+                output.write(encodedAcl);
             }
             try (var channel = java.nio.channels.FileChannel.open(pending.resolve("manifest"), StandardOpenOption.READ)) {
                 channel.force(true);
@@ -97,16 +113,19 @@ final class MultipartStore implements MultipartStorage {
             MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
             long count = 0;
             try (var output = Files.newOutputStream(pending)) {
-                byte[] buffer = new byte[65536]; int n;
+                byte[] buffer = new byte[65536];
+                int n;
                 while ((n = input.read(buffer)) != -1) {
                     count += n;
                     if (count > length) throw new StoreException(413, "EntityTooLarge", "Part exceeds declared size");
-                    sha.update(buffer, 0, n); md5.update(buffer, 0, n); output.write(buffer, 0, n);
+                    sha.update(buffer, 0, n);
+                    md5.update(buffer, 0, n);
+                    output.write(buffer, 0, n);
                 }
             }
             if (count != length) throw new StoreException(400, "IncompleteBody", "Part length does not match Content-Length");
             byte[] actual = sha.digest();
-            if (!MessageDigest.isEqual(actual, HexFormat.of().parseHex(expectedHash)))
+            if (expectedHash != null && !MessageDigest.isEqual(actual, HexFormat.of().parseHex(expectedHash)))
                 throw new StoreException(400, "XAmzContentSHA256Mismatch", "Part hash mismatch");
             if (checksum != null && !Base64.getEncoder().encodeToString(actual).equals(checksum))
                 throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch");
@@ -124,7 +143,8 @@ final class MultipartStore implements MultipartStorage {
             throw new StoreException(400, "InvalidPart", "No valid parts supplied");
         MessageDigest sha = digest("SHA-256");
         List<Path> paths = new ArrayList<>();
-        long total = 0; int last = 0;
+        long total = 0;
+        int last = 0;
         for (Part part : parts) {
             if (part.number() <= last || part.number() > 10000)
                 throw new StoreException(400, "InvalidPartOrder", "Parts must be in ascending order");
@@ -136,8 +156,12 @@ final class MultipartStore implements MultipartStorage {
             if (total > store.maxObject()) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit");
             MessageDigest md5 = digest("MD5");
             try (var input = Files.newInputStream(file)) {
-                byte[] buffer = new byte[65536]; int n;
+                byte[] buffer = new byte[65536];
-                while ((n = input.read(buffer)) != -1) { sha.update(buffer, 0, n); md5.update(buffer, 0, n); }
+                int n;
+                while ((n = input.read(buffer)) != -1) {
+                    sha.update(buffer, 0, n);
+                    md5.update(buffer, 0, n);
+                }
             }
             if (!SigV4.hex(md5.digest()).equals(part.etag().replace("\"", "")))
                 throw new StoreException(400, "InvalidPart", "Part ETag mismatch");
@@ -145,8 +169,9 @@ final class MultipartStore implements MultipartStorage {
         }
         ObjectStorage.Metadata result;
         try (InputStream input = new PartsInput(paths)) {
+            Upload upload = readUpload(dir);
             result = store.put(bucket, key, input, total, SigV4.hex(sha.digest()), null, false,
-                readUpload(dir).contentType());
+                upload.contentType(), upload.userMetadata(), upload.tags(), Map::of, upload.acl());
         }
         remove(dir);
         return result;
@@ -156,6 +181,48 @@ final class MultipartStore implements MultipartStorage {
         remove(upload(id, bucket, key));
     }
 
+    @Override public synchronized PartPage listParts(String id, String bucket, String key,
+                                                     int marker, int maxParts) throws IOException {
+        Path dir = upload(id, bucket, key);
+        List<PartInfo> parts = new ArrayList<>();
+        boolean truncated = false;
+        try (var files = Files.list(dir)) {
+            for (Path file : files.filter(path -> path.getFileName().toString().matches("part-[0-9]{5}"))
+                    .sorted().toList()) {
+                int number = Integer.parseInt(file.getFileName().toString().substring(5));
+                if (number <= marker) continue;
+                if (parts.size() == maxParts) {
+                    truncated = true;
+                    break;
+                }
+                MessageDigest md5 = digest("MD5");
+                try (InputStream input = Files.newInputStream(file)) {
+                    byte[] buffer = new byte[65536];
+                    int count;
+                    while ((count = input.read(buffer)) != -1) md5.update(buffer, 0, count);
+                }
+                parts.add(new PartInfo(number, Files.size(file), SigV4.hex(md5.digest()),
+                    Files.getLastModifiedTime(file).toMillis()));
+            }
+        }
+        int next = parts.isEmpty() ? marker : parts.getLast().number();
+        return new PartPage(parts, next, truncated);
+    }
+
+    @Override public synchronized List<UploadInfo> listUploads(String bucket, String prefix) throws IOException {
+        List<UploadInfo> uploads = new ArrayList<>();
+        try (var dirs = Files.list(root)) {
+            for (Path dir : dirs.filter(Files::isDirectory).toList()) {
+                Upload upload = readUpload(dir);
+                if (upload.bucket().equals(bucket) && upload.key().startsWith(prefix))
+                    uploads.add(new UploadInfo(dir.getFileName().toString(), upload.key(),
+                        Files.getLastModifiedTime(dir.resolve("manifest")).toMillis()));
+            }
+        }
+        uploads.sort(Comparator.comparing(UploadInfo::key).thenComparing(UploadInfo::id));
+        return uploads;
+    }
+
     private Path upload(String id, String bucket, String key) throws IOException {
         if (!id.matches("[0-9a-f-]{36}")) throw new StoreException(404, "NoSuchUpload", "Upload not found");
         Path dir = root.resolve(id);
@@ -167,8 +234,14 @@ final class MultipartStore implements MultipartStorage {
     }
     private static Upload readUpload(Path dir) throws IOException {
         try (var input = new DataInputStream(Files.newInputStream(dir.resolve("manifest")))) {
-            if (input.readInt() != MAGIC) throw new IOException("Invalid multipart upload manifest");
+            int magic = input.readInt();
-            Upload upload = new Upload(input.readUTF(), input.readUTF(), input.readUTF());
+            if (magic != MAGIC && magic != MAGIC_V2 && magic != MAGIC_V3)
+                throw new IOException("Invalid multipart upload manifest");
+            String bucket = input.readUTF(), key = input.readUTF(), type = input.readUTF();
+            Map<String, String> custom = magic != MAGIC ? ObjectAttributes.decode(input.readNBytes(input.readUnsignedShort())) : Map.of();
+            Map<String, String> tags = magic != MAGIC ? ObjectAttributes.decode(input.readNBytes(input.readUnsignedShort())) : Map.of();
+            Map<String, String> acl = magic == MAGIC_V3 ? ObjectAttributes.decode(input.readNBytes(input.readUnsignedShort())) : Map.of();
+            Upload upload = new Upload(bucket, key, type, custom, tags, acl);
             if (input.read() != -1) throw new IOException("Invalid multipart upload manifest");
             return upload;
         }
@@ -209,7 +282,8 @@ final class MultipartStore implements MultipartStorage {
                 }
                 int n = current.read(buffer, offset, length);
                 if (n >= 0) return n;
-                current.close(); current = null;
+                current.close();
+                current = null;
             }
         }
         @Override public void close() throws IOException { if (current != null) current.close(); }

src/cloud/lunarsky/store/NodeClient.java

@@ -9,22 +9,33 @@ import java.net.http.HttpResponse;
 import java.security.MessageDigest;
 import java.time.Duration;
 import java.util.HashSet;
+import java.util.ArrayList;
 import java.util.HexFormat;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 
 final class NodeClient {
     record Node(UUID id, UUID hostId, URI url) {}
+    record StoredSegment(UUID id, long modified) {}
 
-    private static final HttpClient IDENTITY_HTTP = HttpClient.newBuilder()
+    private static HttpClient identityHttp;
-        .connectTimeout(Duration.ofSeconds(2)).build();
     private final List<Node> nodes;
     private final String token;
     private final String repairToken;
-    private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(3)).build();
+    private final HttpClient http;
+    private final ConcurrentHashMap<UUID, Long> unreadableUntil = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<UUID, Long> healthyUntil = new ConcurrentHashMap<>();
+    private static final long READ_RETRY_NANOS = TimeUnit.SECONDS.toNanos(5);
+    private static final long HEALTH_FRESH_NANOS = TimeUnit.SECONDS.toNanos(3);
 
-    NodeClient(List<Node> nodes, String token, String repairToken) {
+    NodeClient(List<Node> nodes, String token, String repairToken) throws IOException {
+        this(nodes, token, repairToken, ClusterTls.client(System.getenv(), Duration.ofSeconds(3)));
+    }
+
+    NodeClient(List<Node> nodes, String token, String repairToken, HttpClient http) {
         if (nodes.isEmpty() || nodes.stream().map(Node::id).distinct().count() != nodes.size() ||
             nodes.stream().map(Node::url).distinct().count() != nodes.size())
             throw new IllegalArgumentException("Cluster node IDs and URLs must be unique");
@@ -37,13 +48,17 @@ final class NodeClient {
         this.nodes = List.copyOf(nodes);
         this.token = token;
         this.repairToken = repairToken;
+        this.http = java.util.Objects.requireNonNull(http);
     }
 
     int count() { return nodes.size(); }
     List<Node> nodes() { return nodes; }
     Node node(int index) { return nodes.get(index); }
+    boolean repairTokenUnavailable() { return repairToken == null || repairToken.length() < 32; }
     int index(UUID id) {
-        for (int i = 0; i < nodes.size(); i++) if (nodes.get(i).id().equals(id)) return i;
+        for (int i = 0; i < nodes.size(); i++) {
+            if (nodes.get(i).id().equals(id)) return i;
+        }
         return -1;
     }
     UUID faultDomain(int index, boolean testNodeDomains) {
@@ -52,12 +67,16 @@ final class NodeClient {
     }
 
     static NodeIdentity probe(URI url, String token) throws IOException {
+        return probe(url, token, identityHttp());
+    }
+
+    static NodeIdentity probe(URI url, String token, HttpClient http) throws IOException {
         validateUrl(url);
         if (token == null || token.length() < 32) throw new IllegalArgumentException("Invalid cluster token");
         HttpRequest request = HttpRequest.newBuilder(url.resolve("/identity"))
             .timeout(Duration.ofSeconds(2)).header("X-Cluster-Token", token).GET().build();
         try {
-            HttpResponse<InputStream> response = IDENTITY_HTTP.send(request, HttpResponse.BodyHandlers.ofInputStream());
+            HttpResponse<InputStream> response = http.send(request, HttpResponse.BodyHandlers.ofInputStream());
             try (InputStream body = response.body()) {
                 if (response.statusCode() != 200) throw new IOException("Node identity request failed: " + response.statusCode());
                 byte[] bytes = body.readNBytes(128);
@@ -74,28 +93,70 @@ final class NodeClient {
         }
     }
 
+    static NodeIdentity probeIfAvailable(URI url, String token) {
+        try { return probe(url, token); }
+        catch (IOException offline) { return null; }
+    }
+
+    private static NodeIdentity probeIfAvailable(URI url, String token, HttpClient http) {
+        try { return probe(url, token, http); }
+        catch (IOException offline) { return null; }
+    }
+
+    private static synchronized HttpClient identityHttp() throws IOException {
+        if (identityHttp == null)
+            identityHttp = ClusterTls.client(System.getenv(), Duration.ofSeconds(2));
+        return identityHttp;
+    }
+
     static void validateUrl(URI url) {
-        if (url == null || !"http".equals(url.getScheme()) || url.getHost() == null ||
+        String trustStore = System.getenv("CLUSTER_TLS_TRUSTSTORE");
+        validateUrl(url, trustStore != null && !trustStore.isBlank());
+    }
+
+    static void validateUrl(URI url, boolean requireHttps) {
+        if (url == null || !("http".equals(url.getScheme()) || "https".equals(url.getScheme())) ||
+            url.getHost() == null ||
             url.getPort() < 1 || url.getRawUserInfo() != null ||
             (url.getRawPath() != null && !url.getRawPath().isEmpty()) ||
             url.getRawQuery() != null || url.getRawFragment() != null)
             throw new IllegalArgumentException("Invalid private storage node URL");
+        if (requireHttps && !"https".equals(url.getScheme()))
+            throw new IllegalArgumentException("Cluster TLS truststore requires HTTPS node URLs");
     }
 
     boolean availableHostsAtLeast(int required, boolean testNodeDomains) {
         Set<UUID> healthy = new HashSet<>();
         for (int i = 0; i < nodes.size(); i++) {
             Node node = nodes.get(i);
-            try {
+            NodeIdentity actual = probeIfAvailable(node.url(), token, http);
-                NodeIdentity actual = probe(node.url(), token);
+            if (actual == null || !actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())) {
-                if (actual.nodeId().equals(node.id()) && actual.hostId().equals(node.hostId()))
+                markUnreadable(node);
+                continue;
+            }
+            unreadableUntil.remove(node.id());
+            healthyUntil.put(node.id(), System.nanoTime() + HEALTH_FRESH_NANOS);
             healthy.add(faultDomain(i, testNodeDomains));
             if (healthy.size() >= required) return true;
-            } catch (IOException error) { }
         }
         return false;
     }
 
+    private void markUnreadable(Node node) {
+        healthyUntil.remove(node.id());
+        unreadableUntil.put(node.id(), System.nanoTime() + READ_RETRY_NANOS);
+    }
+
+    private boolean unreadable(Node node) {
+        Long until = unreadableUntil.get(node.id());
+        return until != null && System.nanoTime() - until < 0;
+    }
+
+    private boolean recentlyHealthy(Node node) {
+        Long until = healthyUntil.get(node.id());
+        return until != null && System.nanoTime() - until < 0;
+    }
+
     void put(int index, UUID id, byte[] data, byte[] sha256) throws IOException {
         put(index, id, data, sha256, false);
     }
@@ -123,18 +184,92 @@ final class NodeClient {
     byte[] get(int index, UUID id, int length, byte[] sha256) throws IOException {
         if (length < 1 || length > ClusterNode.MAX_SEGMENT) throw new IOException("Invalid segment length");
         Node node = nodes.get(index);
+        if (unreadable(node)) throw new IOException("Storage node is temporarily unreachable");
+        if (!recentlyHealthy(node)) {
+            NodeIdentity actual = probeIfAvailable(node.url(), token, http);
+            if (actual == null || !actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())) {
+                markUnreadable(node);
+                throw new IOException("Storage node is temporarily unreachable");
+            }
+            healthyUntil.put(node.id(), System.nanoTime() + HEALTH_FRESH_NANOS);
+        }
         HttpRequest request = HttpRequest.newBuilder(node.url().resolve("/segments/" + id))
             .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token)
             .header("X-Cluster-Expected-Node", node.id().toString()).GET().build();
-        HttpResponse<InputStream> response = send(request, HttpResponse.BodyHandlers.ofInputStream());
+        HttpResponse<InputStream> response;
-        try (InputStream body = response.body()) {
+        try { response = send(request, HttpResponse.BodyHandlers.ofInputStream()); }
-            if (response.statusCode() != 200)
+        catch (IOException error) {
+            markUnreadable(node);
+            throw error;
+        }
+        if (response.statusCode() != 200) {
+            response.body().close();
             throw new IOException("Node " + node.id() + " has no verified copy of segment " + id);
-            byte[] bytes = body.readNBytes(length + 1);
+        }
+        byte[] bytes;
+        try (InputStream body = response.body()) { bytes = body.readNBytes(length + 1); }
+        catch (IOException error) {
+            markUnreadable(node);
+            throw error;
+        }
         if (bytes.length != length || !MessageDigest.isEqual(SigV4.hash(bytes), sha256))
             throw new IOException("Node " + node.id() + " has no verified copy of segment " + id);
+        unreadableUntil.remove(node.id());
+        healthyUntil.put(node.id(), System.nanoTime() + HEALTH_FRESH_NANOS);
         return bytes;
     }
+
+    List<StoredSegment> inventory(int index, String shard, UUID after) throws IOException {
+        if (repairToken == null || repairToken.length() < 32)
+            throw new IOException("Repair authority is not available to this process");
+        Node node = nodes.get(index);
+        String path = "/segments?shard=" + shard + (after == null ? "" : "&after=" + after);
+        HttpRequest request = HttpRequest.newBuilder(node.url().resolve(path))
+            .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token)
+            .header("X-Cluster-Expected-Node", node.id().toString())
+            .header("X-Cluster-Repair-Token", repairToken).GET().build();
+        HttpResponse<InputStream> response = send(request, HttpResponse.BodyHandlers.ofInputStream());
+        try (InputStream body = response.body()) {
+            if (response.statusCode() != 200) throw new IOException("Node inventory failed: " + response.statusCode());
+            byte[] bytes = body.readNBytes(70001);
+            if (bytes.length > 70000) throw new IOException("Node inventory response is too large");
+            List<StoredSegment> result = new ArrayList<>();
+            String last = after == null ? "" : after.toString();
+            for (String line : new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).split("\n")) {
+                if (line.isEmpty()) continue;
+                String[] fields = line.split(" ", -1);
+                if (fields.length != 2) throw new IOException("Invalid node inventory response");
+                try {
+                    UUID id = UUID.fromString(fields[0]);
+                    if (!id.toString().equals(fields[0]) || !fields[0].startsWith(shard) ||
+                        fields[0].compareTo(last) <= 0)
+                        throw new IOException("Invalid node inventory cursor");
+                    result.add(new StoredSegment(id, Long.parseLong(fields[1])));
+                    last = fields[0];
+                } catch (IllegalArgumentException error) {
+                    throw new IOException("Invalid node inventory response", error);
+                }
+            }
+            if (result.size() > 1000) throw new IOException("Node inventory page is too large");
+            return result;
+        }
+    }
+
+    boolean deleteOrphan(int index, StoredSegment segment, long minimumAgeMillis) throws IOException {
+        if (repairToken == null || repairToken.length() < 32)
+            throw new IOException("Repair authority is not available to this process");
+        Node node = nodes.get(index);
+        HttpRequest request = HttpRequest.newBuilder(node.url().resolve("/segments/" + segment.id()))
+            .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token)
+            .header("X-Cluster-Expected-Node", node.id().toString())
+            .header("X-Cluster-Repair-Token", repairToken)
+            .header("X-Cluster-Expected-Mtime", Long.toString(segment.modified()))
+            .header("X-Cluster-Gc-Min-Age-Millis", Long.toString(minimumAgeMillis))
+            .DELETE().build();
+        HttpResponse<Void> response = send(request, HttpResponse.BodyHandlers.discarding());
+        if (response.statusCode() == 204) return true;
+        if (response.statusCode() == 404 || response.statusCode() == 409) return false;
+        throw new IOException("Node refused orphan deletion: " + response.statusCode());
     }
 
     private <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> handler) throws IOException {

src/cloud/lunarsky/store/NodeRegistry.java

@@ -52,7 +52,8 @@ final class NodeRegistry {
             connection.commit();
             return new NodeClient.Node(identity.nodeId(), identity.hostId(), url);
         } catch (SQLException | IOException error) {
-            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
+            try { connection.rollback(); }
+            catch (SQLException rollback) { error.addSuppressed(rollback); }
             if (error instanceof IOException io) throw io;
             throw new IOException("Node registration failed", error);
         } finally {
@@ -69,6 +70,26 @@ final class NodeRegistry {
             try (Statement statement = connection.createStatement()) {
                 statement.execute("SELECT pg_advisory_xact_lock(6834071092781)");
             }
+            Map<String, NodeClient.Node> stored = registeredNodes(connection);
+            if (stored.isEmpty()) registerInitialNodes(connection, urls, token, stored);
+            List<NodeClient.Node> configured = configuredNodes(urls, token, stored);
+            ensureLiveReplicasConfigured(connection, configured);
+            NodeClient nodes = new NodeClient(configured, token, repairToken);
+            connection.commit();
+            return nodes;
+        } catch (SQLException | IOException | RuntimeException error) {
+            try { connection.rollback(); }
+            catch (SQLException rollback) { error.addSuppressed(rollback); }
+            if (error instanceof IOException io) throw io;
+            if (error instanceof SQLException sql) throw new IOException("Node registry check failed", sql);
+            throw (RuntimeException) error;
+        } finally {
+            try { connection.setAutoCommit(true); }
+            catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); }
+        }
+    }
+
+    private static Map<String, NodeClient.Node> registeredNodes(Connection connection) throws SQLException {
         Map<String, NodeClient.Node> stored = new HashMap<>();
         try (Statement statement = connection.createStatement();
              ResultSet result = statement.executeQuery("SELECT node_id, host_id, endpoint FROM cluster_nodes WHERE state <> 'retired'")) {
@@ -78,7 +99,11 @@ final class NodeRegistry {
                     (UUID) result.getObject(2), url));
             }
         }
-            if (stored.isEmpty()) {
+        return stored;
+    }
+
+    private static void registerInitialNodes(Connection connection, List<URI> urls, String token,
+                                             Map<String, NodeClient.Node> stored) throws SQLException, IOException {
         try (Statement statement = connection.createStatement();
              ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_segments)")) {
             result.next();
@@ -96,20 +121,25 @@ final class NodeRegistry {
             stored.put(url.toString(), new NodeClient.Node(identity.nodeId(), identity.hostId(), url));
         }
     }
+
+    private static List<NodeClient.Node> configuredNodes(List<URI> urls, String token,
+                                                         Map<String, NodeClient.Node> stored) throws IOException {
         List<NodeClient.Node> configured = new ArrayList<>();
-            Set<UUID> configuredIds = new HashSet<>();
         for (URI url : urls) {
             NodeClient.Node node = stored.get(url.toString());
             if (node == null) throw new IOException("Unregistered storage node URL: " + url);
-                NodeIdentity actual = null;
+            NodeIdentity actual = NodeClient.probeIfAvailable(url, token);
-                try {
-                    actual = NodeClient.probe(url, token);
-                } catch (IOException offline) { }
             if (actual != null && (!actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())))
                 throw new IOException("Storage node identity changed at " + url);
             configured.add(node);
-                configuredIds.add(node.id());
         }
+        return configured;
+    }
+
+    private static void ensureLiveReplicasConfigured(Connection connection, List<NodeClient.Node> configured)
+        throws SQLException, IOException {
+        Set<UUID> configuredIds = new HashSet<>();
+        for (NodeClient.Node node : configured) configuredIds.add(node.id());
         try (Statement statement = connection.createStatement();
              ResultSet result = statement.executeQuery(
                  "SELECT DISTINCT unnest(s.replica_ids) FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation")) {
@@ -119,17 +149,5 @@ final class NodeRegistry {
                     throw new IOException("A live segment refers to a node missing from CLUSTER_NODES: " + id);
             }
         }
-            NodeClient nodes = new NodeClient(configured, token, repairToken);
-            connection.commit();
-            return nodes;
-        } catch (SQLException | IOException | RuntimeException error) {
-            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
-            if (error instanceof IOException io) throw io;
-            if (error instanceof SQLException sql) throw new IOException("Node registry check failed", sql);
-            throw (RuntimeException) error;
-        } finally {
-            try { connection.setAutoCommit(true); }
-            catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); }
-        }
     }
 }

src/cloud/lunarsky/store/ObjectAttributes.java

@@ -0,0 +1,114 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.Headers;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.TreeMap;
+
+final class ObjectAttributes {
+    private ObjectAttributes() {}
+
+    static Map<String, String> userMetadata(Headers headers) {
+        Map<String, String> values = new TreeMap<>();
+        int total = 0;
+        for (String name : headers.keySet()) {
+            if (!name.toLowerCase(java.util.Locale.ROOT).startsWith("x-amz-meta-")) continue;
+            String key = name.substring(11).toLowerCase(java.util.Locale.ROOT);
+            if (!key.matches("[a-z0-9][a-z0-9._-]{0,127}"))
+                throw new StoreException(400, "InvalidArgument", "Invalid user metadata key");
+            String value = SigV4.single(headers, name);
+            if (value == null || !value.chars().allMatch(c -> c >= 32 && c <= 126))
+                throw new StoreException(400, "InvalidArgument", "Invalid user metadata value");
+            total += key.getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length;
+            values.put(key, value);
+        }
+        if (total > 2048) throw new StoreException(400, "MetadataTooLarge", "User metadata exceeds 2 KiB");
+        return Map.copyOf(values);
+    }
+
+    static Map<String, String> tagsHeader(String raw) {
+        if (raw == null) return Map.of();
+        Map<String, String> tags = new TreeMap<>();
+        if (raw.isEmpty()) return tags;
+        for (String pair : raw.split("&", -1)) {
+            String[] parts = pair.split("=", 2);
+            if (parts.length != 2) throw new StoreException(400, "InvalidTag", "Invalid tagging header");
+            String key = SigV4.decode(parts[0]);
+            String value = SigV4.decode(parts[1]);
+            if (tags.put(key, value) != null) throw new StoreException(400, "InvalidTag", "Duplicate tag key");
+        }
+        validateTags(tags);
+        return Map.copyOf(tags);
+    }
+
+    static void validateTags(Map<String, String> tags) {
+        if (tags.size() > 10) throw new StoreException(400, "InvalidTag", "Too many object tags");
+        for (var entry : tags.entrySet()) {
+            if (entry.getKey().isEmpty() || entry.getKey().length() > 128 ||
+                entry.getValue().length() > 256 ||
+                !xmlText(entry.getKey()) || !xmlText(entry.getValue()))
+                throw new StoreException(400, "InvalidTag", "Invalid tag key or value");
+        }
+    }
+
+    private static boolean xmlText(String value) {
+        for (int i = 0; i < value.length();) {
+            int point = value.codePointAt(i);
+            if (point < 32 || point > 0x10ffff || point >= 0xd800 && point <= 0xdfff ||
+                point >= 0xfffe && point <= 0xffff) return false;
+            i += Character.charCount(point);
+        }
+        return true;
+    }
+
+    static byte[] encode(Map<String, String> values, int limit) {
+        try {
+            ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+            DataOutputStream output = new DataOutputStream(bytes);
+            output.writeShort(values.size());
+            for (var entry : new TreeMap<>(values).entrySet()) {
+                byte[] key = entry.getKey().getBytes(StandardCharsets.UTF_8);
+                byte[] value = entry.getValue().getBytes(StandardCharsets.UTF_8);
+                output.writeShort(key.length);
+                output.writeShort(value.length);
+                output.write(key);
+                output.write(value);
+            }
+            if (bytes.size() > limit) throw new StoreException(400, "InvalidArgument", "Object attributes are too large");
+            return bytes.toByteArray();
+        } catch (IOException error) { throw new IllegalStateException(error); }
+    }
+
+    static Map<String, String> decode(byte[] bytes) throws IOException {
+        if (bytes == null || bytes.length == 0) return Map.of();
+        try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes))) {
+            int count = input.readUnsignedShort();
+            if (count > 128) throw new IOException("Invalid object attributes");
+            Map<String, String> values = new TreeMap<>();
+            for (int i = 0; i < count; i++) {
+                int keyLength = input.readUnsignedShort(), valueLength = input.readUnsignedShort();
+                String key = decodeUtf8(input.readNBytes(keyLength), keyLength);
+                String value = decodeUtf8(input.readNBytes(valueLength), valueLength);
+                if (values.put(key, value) != null) throw new IOException("Duplicate object attribute");
+            }
+            if (input.available() != 0) throw new IOException("Trailing object attributes");
+            return Map.copyOf(values);
+        }
+    }
+
+    private static String decodeUtf8(byte[] bytes, int length) throws IOException {
+        if (bytes.length != length) throw new IOException("Truncated object attributes");
+        try {
+            return StandardCharsets.UTF_8.newDecoder()
+                .onMalformedInput(java.nio.charset.CodingErrorAction.REPORT)
+                .decode(java.nio.ByteBuffer.wrap(bytes)).toString();
+        } catch (java.nio.charset.CharacterCodingException error) {
+            throw new IOException("Invalid object attributes", error);
+        }
+    }
+}

src/cloud/lunarsky/store/ObjectStorage.java

@@ -3,11 +3,46 @@ package cloud.lunarsky.store;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Map;
 
 /** Storage operations shared by the local and cluster gateways. */
 interface ObjectStorage extends AutoCloseable {
+    enum VersioningState { NEVER, ENABLED, SUSPENDED }
+    record Limits(long maxObjectBytes, long maxTotalBytes) {}
+    record Bucket(String name, long created, VersioningState versioning, Map<String, String> acl) {
+        Bucket(String name, long created) { this(name, created, VersioningState.NEVER, Map.of()); }
+        Bucket(String name, long created, VersioningState versioning) {
+            this(name, created, versioning, Map.of());
+        }
+    }
     record Metadata(long length, long modified, String etag, byte[] sha256,
-                    String bucket, String key, String contentType) {}
+                    String bucket, String key, String contentType,
+                    Map<String, String> userMetadata, Map<String, String> tags,
+                    String versionId, Map<String, String> checksums, Map<String, String> acl) {
+        Metadata(long length, long modified, String etag, byte[] sha256,
+                 String bucket, String key, String contentType,
+                 Map<String, String> userMetadata, Map<String, String> tags,
+                 String versionId, Map<String, String> checksums) {
+            this(length, modified, etag, sha256, bucket, key, contentType,
+                userMetadata, tags, versionId, checksums, Map.of());
+        }
+        Metadata(long length, long modified, String etag, byte[] sha256,
+                 String bucket, String key, String contentType,
+                 Map<String, String> userMetadata, Map<String, String> tags,
+                 String versionId) {
+            this(length, modified, etag, sha256, bucket, key, contentType,
+                userMetadata, tags, versionId, Map.of());
+        }
+        Metadata(long length, long modified, String etag, byte[] sha256,
+                 String bucket, String key, String contentType,
+                 Map<String, String> userMetadata, Map<String, String> tags) {
+            this(length, modified, etag, sha256, bucket, key, contentType, userMetadata, tags, null);
+        }
+        Metadata(long length, long modified, String etag, byte[] sha256,
+                 String bucket, String key, String contentType) {
+            this(length, modified, etag, sha256, bucket, key, contentType, Map.of(), Map.of(), null);
+        }
+    }
     record OpenObject(Metadata metadata, InputStream stream) implements AutoCloseable {
         public void close() throws IOException { stream.close(); }
     }
@@ -15,12 +50,72 @@ interface ObjectStorage extends AutoCloseable {
     record ListPage(List<ListedObject> objects, List<String> prefixes, String nextKey, boolean truncated) {
         int keyCount() { return objects.size() + prefixes.size(); }
     }
+    record VersionEntry(String key, String versionId, long modified, boolean deleteMarker,
+                        boolean latest, Metadata metadata) {}
+    record VersionPage(List<VersionEntry> entries, String nextKey, String nextVersionId,
+                       boolean truncated) {}
+    record DeleteResult(String versionId, boolean deleteMarker) {}
 
     Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
-                 String checksum, boolean createOnly, String contentType) throws IOException;
+                 String checksum, boolean createOnly, String contentType,
+                 Map<String, String> userMetadata, Map<String, String> tags,
+                 java.util.function.Supplier<Map<String, String>> checksums,
+                 Map<String, String> acl) throws IOException;
+    default Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
+                         String checksum, boolean createOnly, String contentType,
+                         Map<String, String> userMetadata, Map<String, String> tags,
+                         java.util.function.Supplier<Map<String, String>> checksums) throws IOException {
+        return put(bucket, key, input, length, expectedHash, checksum, createOnly, contentType,
+            userMetadata, tags, checksums, Map.of());
+    }
+    default Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
+                         String checksum, boolean createOnly, String contentType,
+                         Map<String, String> userMetadata, Map<String, String> tags) throws IOException {
+        return put(bucket, key, input, length, expectedHash, checksum, createOnly, contentType,
+            userMetadata, tags, Map::of);
+    }
+    default Metadata put(String bucket, String key, InputStream input, long length, String expectedHash,
+                         String checksum, boolean createOnly, String contentType) throws IOException {
+        return put(bucket, key, input, length, expectedHash, checksum, createOnly, contentType, Map.of(), Map.of());
+    }
     OpenObject open(String bucket, String key) throws IOException;
+    default OpenObject open(String bucket, String key, String versionId) throws IOException {
+        if (versionId == null) return open(bucket, key);
+        throw new StoreException(501, "NotImplemented", "Object versioning is unavailable");
+    }
+    Map<String, String> tags(String bucket, String key) throws IOException;
+    default Map<String, String> tags(String bucket, String key, String versionId) throws IOException {
+        if (versionId == null) return tags(bucket, key);
+        throw new StoreException(501, "NotImplemented", "Versioned tagging is unavailable");
+    }
+    void setTags(String bucket, String key, Map<String, String> tags) throws IOException;
+    default void setTags(String bucket, String key, String versionId,
+                         Map<String, String> tags) throws IOException {
+        if (versionId == null) setTags(bucket, key, tags);
+        else throw new StoreException(501, "NotImplemented", "Versioned tagging is unavailable");
+    }
     void delete(String bucket, String key) throws IOException;
+    default DeleteResult delete(String bucket, String key, String versionId) throws IOException {
+        if (versionId != null) throw new StoreException(501, "NotImplemented", "Object versioning is unavailable");
+        delete(bucket, key);
+        return new DeleteResult(null, false);
+    }
+    default void setVersioning(String bucket, VersioningState state) throws IOException {
+        throw new StoreException(501, "NotImplemented", "Object versioning is unavailable");
+    }
+    default VersionPage listVersions(String bucket, String prefix, String keyMarker,
+                                     String versionMarker, int maxKeys) throws IOException {
+        throw new StoreException(501, "NotImplemented", "Object versioning is unavailable");
+    }
     ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) throws IOException;
+    void ensureBucket(String bucket) throws IOException;
+    Bucket bucket(String bucket) throws IOException;
+    List<Bucket> buckets() throws IOException;
+    void createBucket(String bucket) throws IOException;
+    void deleteBucket(String bucket) throws IOException;
+    void setBucketAcl(String bucket, Map<String, String> acl) throws IOException;
+    void setObjectAcl(String bucket, String key, String versionId, Map<String, String> acl) throws IOException;
+    Limits limits();
     default boolean ready() { return true; }
     void close() throws IOException;
 }

src/cloud/lunarsky/store/SchemaMigrator.java

@@ -21,7 +21,7 @@ final class SchemaMigrator {
                     result.next();
                     version = result.getInt(1);
                 }
-                if (version > 2) throw new IOException("Metadata schema is newer than this ObjectStore build");
+                if (version > 10) throw new IOException("Metadata schema is newer than this ObjectStore build");
                 if (version < 1) {
                     statement.execute("CREATE TABLE IF NOT EXISTS cluster_usage (bucket text PRIMARY KEY, used_bytes bigint NOT NULL CHECK (used_bytes >= 0))");
                     statement.execute("CREATE TABLE IF NOT EXISTS cluster_objects (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, length bigint NOT NULL, modified bigint NOT NULL, etag text NOT NULL, sha256 bytea NOT NULL, content_type text NOT NULL, PRIMARY KEY (bucket, object_key))");
@@ -37,12 +37,79 @@ final class SchemaMigrator {
                     statement.execute("CREATE TABLE IF NOT EXISTS cluster_format (singleton integer PRIMARY KEY CHECK (singleton=1), version integer NOT NULL)");
                     statement.execute("INSERT INTO cluster_schema_migrations VALUES (2)");
                 }
+                if (version < 3) {
+                    statement.execute("CREATE TABLE cluster_uploads (upload_id uuid PRIMARY KEY, bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, content_type text NOT NULL, created_at bigint NOT NULL)");
+                    statement.execute("CREATE TABLE cluster_upload_parts (upload_id uuid NOT NULL REFERENCES cluster_uploads(upload_id) ON DELETE CASCADE, part_number integer NOT NULL CHECK (part_number BETWEEN 1 AND 10000), length bigint NOT NULL CHECK (length >= 0), etag text NOT NULL, modified bigint NOT NULL, PRIMARY KEY (upload_id, part_number))");
+                    statement.execute("CREATE TABLE cluster_upload_segments (upload_id uuid NOT NULL, part_number integer NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replica_ids uuid[] NOT NULL, placement_version bigint NOT NULL DEFAULT 0, PRIMARY KEY (upload_id, part_number, ordinal), FOREIGN KEY (upload_id, part_number) REFERENCES cluster_upload_parts(upload_id, part_number) ON DELETE CASCADE)");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (3)");
+                }
+                if (version < 4) {
+                    statement.execute("CREATE TABLE cluster_gc_candidates (node_id uuid NOT NULL, segment_id uuid NOT NULL, observed_mtime bigint NOT NULL, first_seen bigint NOT NULL, PRIMARY KEY (node_id, segment_id))");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (4)");
+                }
+                if (version < 5) {
+                    statement.execute("ALTER TABLE cluster_objects ADD COLUMN user_metadata bytea");
+                    statement.execute("ALTER TABLE cluster_objects ADD COLUMN tags bytea");
+                    statement.execute("ALTER TABLE cluster_uploads ADD COLUMN user_metadata bytea");
+                    statement.execute("ALTER TABLE cluster_uploads ADD COLUMN tags bytea");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (5)");
+                }
+                if (version < 6) {
+                    statement.execute("CREATE TABLE cluster_buckets (name text PRIMARY KEY, created_at bigint NOT NULL)");
+                    statement.execute("INSERT INTO cluster_buckets SELECT DISTINCT bucket, " +
+                        "CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS bigint) FROM cluster_usage");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (6)");
+                }
+                if (version < 7) {
+                    statement.execute("ALTER TABLE cluster_buckets ADD COLUMN versioning_state text NOT NULL " +
+                        "DEFAULT 'NEVER' CHECK (versioning_state IN ('NEVER', 'ENABLED', 'SUSPENDED'))");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (7)");
+                }
+                if (version < 8) {
+                    statement.execute("CREATE TABLE cluster_object_versions (sequence bigint GENERATED ALWAYS AS IDENTITY, " +
+                        "bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, version_id text NOT NULL, " +
+                        "delete_marker boolean NOT NULL, generation uuid, length bigint, modified bigint NOT NULL, " +
+                        "etag text, sha256 bytea, content_type text, user_metadata bytea, tags bytea, " +
+                        "PRIMARY KEY (bucket, object_key, version_id), " +
+                        "CHECK (delete_marker = (generation IS NULL)))");
+                    statement.execute("CREATE INDEX cluster_versions_order ON cluster_object_versions " +
+                        "(bucket, object_key, sequence DESC)");
+                    statement.execute("CREATE TABLE cluster_object_heads (bucket text NOT NULL, " +
+                        "object_key text COLLATE \"C\" NOT NULL, version_id text NOT NULL, " +
+                        "PRIMARY KEY (bucket, object_key), " +
+                        "FOREIGN KEY (bucket, object_key, version_id) REFERENCES cluster_object_versions " +
+                        "(bucket, object_key, version_id) DEFERRABLE INITIALLY DEFERRED)");
+                    statement.execute("INSERT INTO cluster_object_versions " +
+                        "(bucket, object_key, version_id, delete_marker, generation, length, modified, etag, " +
+                        "sha256, content_type, user_metadata, tags) " +
+                        "SELECT bucket, object_key, 'null', false, generation, length, modified, etag, sha256, " +
+                        "content_type, user_metadata, tags FROM cluster_objects");
+                    statement.execute("INSERT INTO cluster_object_heads " +
+                        "SELECT bucket, object_key, 'null' FROM cluster_objects");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (8)");
+                }
+                if (version < 9) {
+                    statement.execute("ALTER TABLE cluster_object_versions ADD COLUMN checksum_metadata bytea");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (9)");
+                }
+                if (version < 10) {
+                    statement.execute("ALTER TABLE cluster_buckets ADD COLUMN acl bytea");
+                    statement.execute("ALTER TABLE cluster_object_versions ADD COLUMN acl bytea");
+                    statement.execute("ALTER TABLE cluster_uploads ADD COLUMN acl bytea");
+                    statement.execute("INSERT INTO cluster_schema_migrations VALUES (10)");
+                }
                 statement.execute("INSERT INTO cluster_format SELECT 1, CASE WHEN EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL) THEN 1 ELSE 2 END WHERE NOT EXISTS (SELECT 1 FROM cluster_format)");
             }
             try (PreparedStatement insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, 0) ON CONFLICT DO NOTHING")) {
                 insert.setString(1, bucket);
                 insert.executeUpdate();
             }
+            try (PreparedStatement insert = connection.prepareStatement(
+                "INSERT INTO cluster_buckets (name, created_at) VALUES (?, ?) ON CONFLICT DO NOTHING")) {
+                insert.setString(1, bucket);
+                insert.setLong(2, System.currentTimeMillis());
+                insert.executeUpdate();
+            }
             int format;
             try (Statement statement = connection.createStatement();
                  ResultSet result = statement.executeQuery("SELECT version FROM cluster_format WHERE singleton=1")) {
@@ -60,7 +127,8 @@ final class SchemaMigrator {
             connection.commit();
             return format;
         } catch (SQLException | IOException error) {
-            try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
+            try { connection.rollback(); }
+            catch (SQLException rollback) { error.addSuppressed(rollback); }
             if (error instanceof IOException io) throw io;
             throw new IOException("Metadata schema migration failed", error);
         } finally {

src/cloud/lunarsky/store/SigV4.java

@@ -12,48 +12,193 @@ import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
 import java.util.HexFormat;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 import javax.crypto.Mac;
 import javax.crypto.spec.SecretKeySpec;
 
 final class SigV4 {
+    record Verified(String payload, String applicationQuery, byte[] signingKey,
+                    String date, String scope, String signature, String principal) {
+        Verified(String payload, String applicationQuery, byte[] signingKey,
+                 String date, String scope, String signature) {
+            this(payload, applicationQuery, signingKey, date, scope, signature, null);
+        }
+        boolean streaming() { return payload.startsWith("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"); }
+    }
     private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);
     private static final Pattern HEX = Pattern.compile("[0-9a-f]{64}");
-    private final String accessKey, secretKey, region;
+    private final Map<String, String> identities;
+    private final String root;
+    private final String region;
     private final Clock clock;
 
     SigV4(String accessKey, String secretKey, String region, Clock clock) {
-        this.accessKey = accessKey; this.secretKey = secretKey; this.region = region; this.clock = clock;
+        this(Map.of(accessKey, secretKey), accessKey, region, clock);
     }
 
+    SigV4(Map<String, String> identities, String root, String region, Clock clock) {
+        this.identities = Map.copyOf(identities);
+        if (!this.identities.containsKey(root)) throw new IllegalArgumentException("Missing root identity");
+        this.root = root;
+        this.region = region;
+        this.clock = clock;
+    }
+
+    String root() { return root; }
+    Set<String> identities() { return identities.keySet(); }
+
     String verify(String method, URI uri, Headers headers) {
+        return verifyRequest(method, uri, headers).payload();
+    }
+
+    Verified verifyRequest(String method, URI uri, Headers headers) {
+        if (hasPresignedQuery(uri.getRawQuery())) return verifyPresigned(method, uri, headers);
+        Map<String, String> fields = authorizationFields(headers);
+        String[] credential = credentialScope(fields.get("Credential"));
+        String date = signingDate(headers, credential[1]);
+        String payload = payloadHash(headers);
+        String signedHeaders = fields.get("SignedHeaders");
+        String canonicalHeaders = canonicalHeaders(headers, signedHeaders,
+            Set.of("host", "x-amz-date", "x-amz-content-sha256"));
+        String canonical = method + "\n" + encode(decode(uri.getRawPath()), true) + "\n"
+            + canonicalQuery(uri.getRawQuery()) + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payload;
+        String scope = String.join("/", Arrays.copyOfRange(credential, 1, 5));
+        String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + hex(hash(canonical.getBytes(StandardCharsets.UTF_8)));
+        byte[] signingKey = signingKey(secret(credential[0]), credential[1], region);
+        String signature = fields.get("Signature");
+        if (!HEX.matcher(signature).matches() || !MessageDigest.isEqual(hmac(signingKey, toSign), HexFormat.of().parseHex(signature))) denied("Signature mismatch");
+        return new Verified(payload, uri.getRawQuery(), signingKey, date, scope, signature, credential[0]);
+    }
+
+    private Verified verifyPresigned(String method, URI uri, Headers headers) {
+        if (headers.containsKey("authorization")) denied("Use one authentication method");
+        PresignedQuery query = presignedQuery(uri.getRawQuery());
+        Map<String, String> fields = query.fields();
+        if (!fields.keySet().equals(Set.of("X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date",
+                "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Signature")) ||
+            !"AWS4-HMAC-SHA256".equals(fields.get("X-Amz-Algorithm")))
+            denied("Invalid presigned parameters");
+        String[] credential = credentialScope(fields.get("X-Amz-Credential"));
+        String date = fields.get("X-Amz-Date");
+        validatePresignedTime(date, credential[1], fields.get("X-Amz-Expires"));
+        String signedHeaders = fields.get("X-Amz-SignedHeaders");
+        String canonicalHeaders = canonicalHeaders(headers, signedHeaders, Set.of("host"));
+        String scope = String.join("/", Arrays.copyOfRange(credential, 1, 5));
+        String canonical = method + "\n" + encode(decode(uri.getRawPath()), true) + "\n"
+            + canonicalQuery(query.signed()) + "\n" + canonicalHeaders + "\n"
+            + signedHeaders + "\nUNSIGNED-PAYLOAD";
+        String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
+            + hex(hash(canonical.getBytes(StandardCharsets.UTF_8)));
+        String signature = fields.get("X-Amz-Signature");
+        byte[] key = signingKey(secret(credential[0]), credential[1], region);
+        if (!HEX.matcher(signature).matches() ||
+            !MessageDigest.isEqual(hmac(key, toSign), HexFormat.of().parseHex(signature)))
+            denied("Signature mismatch");
+        return new Verified("UNSIGNED-PAYLOAD", query.application(), key, date, scope, signature, credential[0]);
+    }
+
+    private static PresignedQuery presignedQuery(String rawQuery) {
+        Map<String, String> fields = new TreeMap<>();
+        StringBuilder application = new StringBuilder();
+        StringBuilder signed = new StringBuilder();
+        for (String part : rawQuery.split("&", -1)) {
+            String[] pair = part.split("=", 2);
+            String name = decode(pair[0]);
+            String value = decode(pair.length == 2 ? pair[1] : "");
+            if (name.startsWith("X-Amz-")) {
+                if (fields.put(name, value) != null) denied("Duplicate presigned parameter");
+                if (!name.equals("X-Amz-Signature")) appendQuery(signed, part);
+            } else {
+                appendQuery(application, part);
+                appendQuery(signed, part);
+            }
+        }
+        return new PresignedQuery(fields, application.toString(), signed.toString());
+    }
+
+    private void validatePresignedTime(String date, String scopeDate, String rawExpires) {
+        if (!date.matches("[0-9]{8}T[0-9]{6}Z") || !date.startsWith(scopeDate))
+            denied("Invalid signing date");
+        long expires;
+        try {
+            expires = Long.parseLong(rawExpires);
+        } catch (NumberFormatException error) {
+            denied("Invalid presigned expiry");
+            return;
+        }
+        if (expires < 1 || expires > 604800) denied("Invalid presigned expiry");
+        try {
+            Instant start = Instant.from(DATE.parse(date));
+            Instant now = clock.instant();
+            if (now.isBefore(start.minus(Duration.ofMinutes(5))) || now.isAfter(start.plusSeconds(expires)))
+                denied("Presigned URL has expired or is not yet valid");
+        } catch (java.time.DateTimeException error) { denied("Invalid signing date"); }
+    }
+
+    private record PresignedQuery(Map<String, String> fields, String application, String signed) { }
+
+    private static boolean hasPresignedQuery(String raw) {
+        return raw != null && (raw.startsWith("X-Amz-Algorithm=") || raw.contains("&X-Amz-Algorithm="));
+    }
+
+    private static void appendQuery(StringBuilder target, String part) {
+        if (!target.isEmpty()) target.append('&');
+        target.append(part);
+    }
+
+    private static Map<String, String> authorizationFields(Headers headers) {
         String authorization = single(headers, "authorization");
         if (authorization == null || !authorization.startsWith("AWS4-HMAC-SHA256 ")) denied("Signed requests are required");
-        Map<String,String> fields = new TreeMap<>();
+        Map<String, String> fields = new TreeMap<>();
         for (String part : authorization.substring(17).split(",")) {
             String[] pair = part.trim().split("=", 2);
             if (pair.length != 2 || fields.put(pair[0], pair[1]) != null) denied("Invalid authorization header");
         }
         if (!fields.keySet().equals(java.util.Set.of("Credential", "SignedHeaders", "Signature"))) denied("Invalid authorization fields");
-        String[] credential = fields.get("Credential").split("/", -1);
+        return fields;
-        if (credential.length != 5 || !credential[0].equals(accessKey) || !credential[2].equals(region)
+    }
+
+    private String[] credentialScope(String value) {
+        String[] credential = value.split("/", -1);
+        if (credential.length != 5 || !identities.containsKey(credential[0]) || !credential[2].equals(region)
             || !credential[3].equals("s3") || !credential[4].equals("aws4_request")) denied("Invalid credential scope");
-        String date = single(headers, "x-amz-date"), payload = single(headers, "x-amz-content-sha256");
+        return credential;
-        if (date == null || !credential[1].matches("[0-9]{8}") || !date.matches("[0-9]{8}T[0-9]{6}Z") || !date.startsWith(credential[1])) denied("Invalid signing date");
+    }
+
+    private String secret(String accessKey) {
+        String secret = identities.get(accessKey);
+        if (secret == null) denied("Invalid credential scope");
+        return secret;
+    }
+
+    private String signingDate(Headers headers, String credentialDate) {
+        String date = single(headers, "x-amz-date");
+        if (date == null || !credentialDate.matches("[0-9]{8}") || !date.matches("[0-9]{8}T[0-9]{6}Z") || !date.startsWith(credentialDate)) denied("Invalid signing date");
         try {
             Instant signed = Instant.from(DATE.parse(date));
             if (Duration.between(signed, clock.instant()).abs().compareTo(Duration.ofMinutes(5)) > 0)
                 throw new StoreException(403, "RequestTimeTooSkewed", "Request timestamp is outside the permitted window");
         } catch (java.time.DateTimeException e) { denied("Invalid signing date"); }
-        if (payload == null || !HEX.matcher(payload).matches())
+        return date;
-            throw new StoreException(400, "NotImplemented", "A hexadecimal SHA-256 payload hash is required; unsigned and chunk-signed payloads are unsupported");
+    }
+
+    private static String payloadHash(Headers headers) {
+        String payload = single(headers, "x-amz-content-sha256");
+        if (payload == null || !(HEX.matcher(payload).matches() ||
+            payload.equals("STREAMING-AWS4-HMAC-SHA256-PAYLOAD") ||
+            payload.equals("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER")))
+            throw new StoreException(400, "NotImplemented", "Unsupported SHA-256 payload mode");
         if (headers.containsKey("x-amz-security-token")) denied("Temporary credentials are unsupported");
-        String signedHeaders = fields.get("SignedHeaders");
+        return payload;
+    }
+
+    private static String canonicalHeaders(Headers headers, String signedHeaders, Set<String> required) {
         String[] names = signedHeaders.split(";", -1);
         if (names.length > 32 || !signedHeaders.equals(String.join(";", Arrays.stream(names).distinct().sorted().toList()))) denied("Signed headers must be unique and sorted");
         var namesSet = java.util.Set.copyOf(Arrays.asList(names));
-        if (!namesSet.containsAll(java.util.Set.of("host", "x-amz-date", "x-amz-content-sha256"))) denied("Missing signed headers");
+        if (!namesSet.containsAll(required)) denied("Missing signed headers");
         for (String key : headers.keySet()) {
             String lower = key.toLowerCase(java.util.Locale.ROOT);
             if (lower.startsWith("x-amz-") && !namesSet.contains(lower)) denied("Unsigned Amazon header");
@@ -66,14 +211,7 @@ final class SigV4 {
             if (value == null) denied("Missing signed header");
             canonicalHeaders.append(name).append(':').append(value.trim().replaceAll("[\\t ]+", " ")).append('\n');
         }
-        String canonical = method + "\n" + encode(decode(uri.getRawPath()), true) + "\n"
+        return canonicalHeaders.toString();
-            + canonicalQuery(uri.getRawQuery()) + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payload;
-        String scope = String.join("/", Arrays.copyOfRange(credential, 1, 5));
-        String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + hex(hash(canonical.getBytes(StandardCharsets.UTF_8)));
-        byte[] signingKey = signingKey(secretKey, credential[1], region);
-        String signature = fields.get("Signature");
-        if (!HEX.matcher(signature).matches() || !MessageDigest.isEqual(hmac(signingKey, toSign), HexFormat.of().parseHex(signature))) denied("Signature mismatch");
-        return payload;
     }
 
     static String single(Headers headers, String name) {
@@ -85,17 +223,25 @@ final class SigV4 {
 
     static String decode(String value) {
         try {
-            var bytes=new java.io.ByteArrayOutputStream();
+            var bytes = new java.io.ByteArrayOutputStream();
-            for(int i=0;i<value.length();){
+            for (int i = 0; i < value.length();) {
-                if(value.charAt(i)=='%'){
+                if (value.charAt(i) == '%') {
-                    if(i+2>=value.length())throw new IllegalArgumentException();
+                    if (i + 2 >= value.length()) throw new IllegalArgumentException();
-                    int hi=Character.digit(value.charAt(i+1),16),lo=Character.digit(value.charAt(i+2),16);
+                    int hi = Character.digit(value.charAt(i + 1), 16);
-                    if(hi<0||lo<0)throw new IllegalArgumentException();
+                    int lo = Character.digit(value.charAt(i + 2), 16);
-                    bytes.write((hi<<4)|lo);i+=3;
+                    if (hi < 0 || lo < 0) throw new IllegalArgumentException();
-                }else{int point=value.codePointAt(i);bytes.writeBytes(new String(Character.toChars(point)).getBytes(StandardCharsets.UTF_8));i+=Character.charCount(point);}
+                    bytes.write((hi << 4) | lo);
+                    i += 3;
+                } else {
+                    int point = value.codePointAt(i);
+                    bytes.writeBytes(new String(Character.toChars(point)).getBytes(StandardCharsets.UTF_8));
+                    i += Character.charCount(point);
+                }
             }
             return StandardCharsets.UTF_8.newDecoder().onMalformedInput(java.nio.charset.CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes.toByteArray())).toString();
-        }catch(IllegalArgumentException|java.nio.charset.CharacterCodingException e){throw new StoreException(400,"InvalidURI","Malformed URI encoding");}
+        } catch (IllegalArgumentException | java.nio.charset.CharacterCodingException e) {
+            throw new StoreException(400, "InvalidURI", "Malformed URI encoding");
+        }
     }
 
     static String encode(String value, boolean keepSlash) {
@@ -120,10 +266,17 @@ final class SigV4 {
         return hmac(hmac(hmac(hmac(("AWS4"+secret).getBytes(StandardCharsets.UTF_8),date),region),"s3"),"aws4_request");
     }
     static byte[] hmac(byte[] key, String text) {
-        try { Mac mac=Mac.getInstance("HmacSHA256");mac.init(new SecretKeySpec(key,"HmacSHA256"));return mac.doFinal(text.getBytes(StandardCharsets.UTF_8)); }
+        try {
+            Mac mac = Mac.getInstance("HmacSHA256");
+            mac.init(new SecretKeySpec(key, "HmacSHA256"));
+            return mac.doFinal(text.getBytes(StandardCharsets.UTF_8));
+        }
         catch (java.security.GeneralSecurityException e) { throw new IllegalStateException(e); }
     }
-    static byte[] hash(byte[] data) { try {return MessageDigest.getInstance("SHA-256").digest(data);}catch(java.security.NoSuchAlgorithmException e){throw new IllegalStateException(e);} }
+    static byte[] hash(byte[] data) {
+        try { return MessageDigest.getInstance("SHA-256").digest(data); }
+        catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
+    }
     static String hex(byte[] data) { return HexFormat.of().formatHex(data); }
     private static void denied(String message) { throw new StoreException(403,"AccessDenied",message); }
 }

src/cloud/lunarsky/store/StorageLimits.java

@@ -0,0 +1,19 @@
+package cloud.lunarsky.store;
+
+import java.util.Map;
+
+final class StorageLimits {
+    private StorageLimits() {}
+
+    static ObjectStorage.Limits fromEnvironment(Map<String, String> environment) {
+        try {
+            long maxObject = Long.parseLong(environment.getOrDefault("MAX_OBJECT_BYTES", "134217728"));
+            long maxTotal = Long.parseLong(environment.getOrDefault("MAX_TOTAL_BYTES", "2147483648"));
+            if (maxObject < 1 || maxObject > 1073741824L || maxTotal < maxObject)
+                throw new NumberFormatException();
+            return new ObjectStorage.Limits(maxObject, maxTotal);
+        } catch (NumberFormatException error) {
+            throw new IllegalArgumentException("Invalid storage size limits", error);
+        }
+    }
+}

src/cloud/lunarsky/store/StoreException.java

@@ -3,6 +3,9 @@ package cloud.lunarsky.store;
 final class StoreException extends RuntimeException {
     final int status;
     final String code;
+    final String versionId;
+    final long modified;
+    final boolean deleteMarker;
     StoreException(int status, String code, String message) {
         this(status, code, message, null);
     }
@@ -10,5 +13,20 @@ final class StoreException extends RuntimeException {
         super(message, cause);
         this.status = status;
         this.code = code;
+        this.versionId = null;
+        this.modified = -1;
+        this.deleteMarker = false;
+    }
+    private StoreException(int status, String code, String message, String versionId, long modified) {
+        super(message);
+        this.status = status;
+        this.code = code;
+        this.versionId = versionId;
+        this.modified = modified;
+        this.deleteMarker = true;
+    }
+    static StoreException deletedVersion(String versionId, long modified, boolean explicit) {
+        return new StoreException(explicit ? 405 : 404, explicit ? "MethodNotAllowed" : "NoSuchKey",
+            "Object is deleted", versionId, modified);
     }
 }

src/cloud/lunarsky/store/UnavailableMultipart.java

@@ -1,17 +0,0 @@
-package cloud.lunarsky.store;
-
-import java.io.InputStream;
-import java.util.List;
-
-final class UnavailableMultipart implements MultipartStorage {
-    private StoreException unavailable() {
-        return new StoreException(501, "NotImplemented", "Multipart uploads are unavailable in the local cluster prototype");
-    }
-    @Override public String create(String bucket, String key, String contentType) { throw unavailable(); }
-    @Override public String putPart(String id, String bucket, String key, int number, InputStream input,
-                                    long length, String expectedHash, String checksum) { throw unavailable(); }
-    @Override public ObjectStorage.Metadata complete(String id, String bucket, String key, List<Part> parts) { throw unavailable(); }
-    @Override public void abort(String id, String bucket, String key) { throw unavailable(); }
-    @Override public int activeUploads() { return 0; }
-    @Override public long stagedBytes() { return 0; }
-}

src/cloud/lunarsky/store/UploadChecksums.java

@@ -0,0 +1,167 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.Headers;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import java.util.Locale;
+import java.util.zip.CRC32;
+import java.util.zip.CRC32C;
+import java.util.zip.Checksum;
+
+final class UploadChecksums {
+    private record Algorithm(String name, String header, int length) {}
+
+    private final byte[] contentMd5;
+    private final Algorithm algorithm;
+    private final byte[] expected;
+    private final String encoded;
+
+    private UploadChecksums(byte[] contentMd5, Algorithm algorithm, byte[] expected, String encoded) {
+        this.contentMd5 = contentMd5;
+        this.algorithm = algorithm;
+        this.expected = expected;
+        this.encoded = encoded;
+    }
+
+    static UploadChecksums from(Headers headers) {
+        String md5 = SigV4.single(headers, "content-md5");
+        byte[] contentMd5 = md5 == null ? null : decode(md5, 16);
+        Algorithm algorithm = null;
+        String encoded = null;
+        for (String name : headers.keySet()) {
+            String lower = name.toLowerCase(Locale.ROOT);
+            if (!lower.startsWith("x-amz-checksum-")) continue;
+            if (algorithm != null)
+                throw new StoreException(400, "InvalidRequest", "Supply one checksum algorithm");
+            algorithm = algorithm(lower);
+            encoded = SigV4.single(headers, name);
+        }
+        String selected = SigV4.single(headers, "x-amz-sdk-checksum-algorithm");
+        if (selected != null) {
+            String trailer = SigV4.single(headers, "x-amz-trailer");
+            if (algorithm == null && trailer != null) {
+                Algorithm declared = algorithm(trailer);
+                if (!selected.equals(declared.name()))
+                    throw new StoreException(400, "InvalidRequest", "Checksum algorithm and trailer must match");
+            } else if (algorithm == null || !selected.equals(algorithm.name())) {
+                throw new StoreException(400, "InvalidRequest", "Checksum algorithm and value must match");
+            }
+        }
+        byte[] expected = algorithm == null ? null : decode(encoded, algorithm.length());
+        return new UploadChecksums(contentMd5, algorithm, expected, encoded);
+    }
+
+    private static Algorithm algorithm(String header) {
+        return switch (header) {
+            case "x-amz-checksum-crc32" -> new Algorithm("CRC32", header, 4);
+            case "x-amz-checksum-crc32c" -> new Algorithm("CRC32C", header, 4);
+            case "x-amz-checksum-crc64nvme" -> new Algorithm("CRC64NVME", header, 8);
+            case "x-amz-checksum-xxhash64" -> new Algorithm("XXHASH64", header, 8);
+            case "x-amz-checksum-xxhash3" -> new Algorithm("XXHASH3", header, 8);
+            case "x-amz-checksum-xxhash128" -> new Algorithm("XXHASH128", header, 16);
+            case "x-amz-checksum-sha1" -> new Algorithm("SHA1", header, 20);
+            case "x-amz-checksum-sha256" -> new Algorithm("SHA256", header, 32);
+            case "x-amz-checksum-sha512" -> new Algorithm("SHA512", header, 64);
+            case "x-amz-checksum-md5" -> new Algorithm("MD5", header, 16);
+            default -> throw new StoreException(501, "NotImplemented", "Checksum algorithm is unsupported");
+        };
+    }
+
+    private static byte[] decode(String value, int length) {
+        try {
+            byte[] decoded = Base64.getDecoder().decode(value);
+            if (decoded.length == length) return decoded;
+        } catch (IllegalArgumentException ignored) { }
+        throw new StoreException(400, "InvalidDigest", "Invalid checksum encoding or length");
+    }
+
+    String sha256() {
+        return algorithm != null && algorithm.name().equals("SHA256") ? encoded : null;
+    }
+
+    java.util.Map<String, String> metadata() {
+        return algorithm == null ? java.util.Map.of() : java.util.Map.of(algorithm.header(), encoded);
+    }
+
+    void response(Headers headers) {
+        if (algorithm != null) headers.set(algorithm.header(), encoded);
+    }
+
+    InputStream verifying(InputStream input) {
+        if (contentMd5 == null && (algorithm == null || algorithm.name().equals("SHA256"))) return input;
+        return new VerifiedInput(input);
+    }
+
+    private final class VerifiedInput extends FilterInputStream {
+        private final MessageDigest md5 = contentMd5 != null ||
+            (algorithm != null && algorithm.name().equals("MD5")) ? digest("MD5") : null;
+        private final MessageDigest hash = algorithm == null ? null : switch (algorithm.name()) {
+            case "SHA1" -> digest("SHA-1");
+            case "SHA512" -> digest("SHA-512");
+            default -> null;
+        };
+        private final Checksum crc = algorithm == null ? null : switch (algorithm.name()) {
+            case "CRC32" -> new CRC32();
+            case "CRC32C" -> new CRC32C();
+            case "CRC64NVME" -> new Crc64Nvme();
+            default -> null;
+        };
+        private final XxHashes xxhash = algorithm != null && algorithm.name().startsWith("XXHASH")
+            ? new XxHashes(algorithm.name()) : null;
+        private boolean checked;
+
+        private VerifiedInput(InputStream input) { super(input); }
+
+        @Override public int read() throws IOException {
+            int value = in.read();
+            if (value < 0) verify();
+            else update(new byte[]{(byte) value}, 0, 1);
+            return value;
+        }
+
+        @Override public int read(byte[] bytes, int offset, int length) throws IOException {
+            int count = in.read(bytes, offset, length);
+            if (count < 0) verify();
+            else if (count > 0) update(bytes, offset, count);
+            return count;
+        }
+
+        private void update(byte[] bytes, int offset, int length) {
+            if (md5 != null) md5.update(bytes, offset, length);
+            if (hash != null) hash.update(bytes, offset, length);
+            if (crc != null) crc.update(bytes, offset, length);
+            if (xxhash != null) xxhash.update(bytes, offset, length);
+        }
+
+        private void verify() {
+            if (checked) return;
+            checked = true;
+            byte[] actualMd5 = md5 == null ? null : md5.digest();
+            if (contentMd5 != null && !MessageDigest.isEqual(contentMd5, actualMd5))
+                throw new StoreException(400, "BadDigest", "Content-MD5 mismatch");
+            if (algorithm == null || algorithm.name().equals("SHA256")) return;
+            byte[] actual;
+            if (crc != null) {
+                long value = crc.getValue();
+                actual = new byte[algorithm.length()];
+                for (int i = actual.length - 1; i >= 0; i--) {
+                    actual[i] = (byte) value;
+                    value >>>= 8;
+                }
+            } else if (algorithm.name().equals("MD5")) actual = actualMd5;
+            else if (xxhash != null) actual = xxhash.digest();
+            else actual = hash.digest();
+            if (!MessageDigest.isEqual(expected, actual))
+                throw new StoreException(400, "BadDigest", algorithm.name() + " checksum mismatch");
+        }
+    }
+
+    private static MessageDigest digest(String algorithm) {
+        try { return MessageDigest.getInstance(algorithm); }
+        catch (NoSuchAlgorithmException error) { throw new IllegalStateException(error); }
+    }
+}

src/cloud/lunarsky/store/Version.java

@@ -1,7 +1,7 @@
 package cloud.lunarsky.store;
 
 final class Version {
-    static final String VALUE = "0.0.2";
+    static final String VALUE = "0.0.8";
 
     private Version() {}
 }

src/cloud/lunarsky/store/XxHashes.java

@@ -0,0 +1,32 @@
+package cloud.lunarsky.store;
+
+import com.dynatrace.hash4j.hashing.HashStream64;
+import com.dynatrace.hash4j.hashing.HashStream128;
+import com.dynatrace.hash4j.hashing.Hashing;
+import java.nio.ByteBuffer;
+
+final class XxHashes {
+    private final HashStream64 stream;
+
+    XxHashes(String algorithm) {
+        stream = switch (algorithm) {
+            case "XXHASH64" -> Hashing.xxh64().hashStream();
+            case "XXHASH3" -> Hashing.xxh3_64().hashStream();
+            case "XXHASH128" -> Hashing.xxh3_128().hashStream();
+            default -> throw new IllegalArgumentException(algorithm);
+        };
+    }
+
+    void update(byte[] bytes, int offset, int length) {
+        stream.putBytes(bytes, offset, length);
+    }
+
+    byte[] digest() {
+        if (stream instanceof HashStream128 wide) {
+            var value = wide.get();
+            return ByteBuffer.allocate(16).putLong(value.getMostSignificantBits())
+                .putLong(value.getLeastSignificantBits()).array();
+        }
+        return ByteBuffer.allocate(8).putLong(stream.getAsLong()).array();
+    }
+}

test/cloud/lunarsky/store/CliTest.java

@@ -44,6 +44,21 @@ public final class CliTest {
             int status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), new PrintStream(output));
             if (status != 1 || !output.toString().contains("Object checksum mismatch"))
                 throw new AssertionError("CLI missed corrupted payload");
+            Path versionRoot = root.resolve("versioned");
+            try (var store = new DiskStore(versionRoot, 100, 1000)) {
+                store.createBucket("versioned-bucket");
+                store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.ENABLED);
+                for (byte[] body : new byte[][]{"first".getBytes(StandardCharsets.UTF_8),
+                    "second".getBytes(StandardCharsets.UTF_8)}) {
+                    store.put("versioned-bucket", "example", new ByteArrayInputStream(body), body.length,
+                        SigV4.hex(SigV4.hash(body)), null, false, "text/plain");
+                }
+                output.reset();
+                status = Cli.run(new String[]{"verify"}, versionRoot,
+                    new PrintStream(output), new PrintStream(output));
+                if (status != 0 || !output.toString().contains("verified_objects=2"))
+                    throw new AssertionError("CLI did not inspect retained versions");
+            }
             System.out.println("CLI tests passed: version, live status, verification, corruption exit code");
         } finally {
             try (var paths = Files.walk(root)) {

test/cloud/lunarsky/store/ClientLimitsTest.java

@@ -0,0 +1,98 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpServer;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.concurrent.Executors;
+
+public final class ClientLimitsTest {
+    private static final String ACCESS = "TESTACCESSKEY123";
+    private static final String SECRET = "test-secret-key-that-is-at-least-32-characters";
+
+    private static HttpResponse<String> get(HttpClient client, String base, String path, String ip) throws Exception {
+        var request = HttpRequest.newBuilder(URI.create(base + path));
+        if (ip != null) request.header("X-Real-IP", ip);
+        return client.send(request.GET().build(), HttpResponse.BodyHandlers.ofString());
+    }
+
+    private static void status(int wanted, HttpResponse<?> response) {
+        if (response.statusCode() != wanted)
+            throw new AssertionError("Expected " + wanted + ", got " + response.statusCode() + ": " + response.body());
+    }
+
+    private static void exercise(Map<String, String> configuration, boolean trusted) throws Exception {
+        Path root = Files.createTempDirectory("client-limits-test-");
+        var executor = Executors.newVirtualThreadPerTaskExecutor();
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16);
+        DiskStore store = new DiskStore(root, 1024, 4096);
+        try {
+            var app = new Main(store, new MultipartStore(store),
+                new SigV4(Map.of(ACCESS, SECRET), ACCESS, "us-east-1", Clock.systemUTC()),
+                "objects", ClientLimits.fromEnvironment(configuration));
+            server.setExecutor(executor);
+            server.createContext("/", app::handle);
+            server.start();
+            String base = "http://127.0.0.1:" + server.getAddress().getPort();
+            HttpClient client = HttpClient.newHttpClient();
+            if (trusted) {
+                status(200, get(client, base, "/health", null));
+                status(400, get(client, base, "/ready", null));
+                status(400, get(client, base, "/objects/example", null));
+                status(400, get(client, base, "/health", "not-an-ip"));
+                status(400, client.send(HttpRequest.newBuilder(URI.create(base + "/health"))
+                    .header("X-Real-IP", "192.0.2.1")
+                    .header("X-Real-IP", "192.0.2.2")
+                    .GET().build(), HttpResponse.BodyHandlers.ofString()));
+                status(200, get(client, base, "/health", "192.0.2.1"));
+                var limited = get(client, base, "/health", "192.0.2.1");
+                status(503, limited);
+                if (!"1".equals(limited.headers().firstValue("Retry-After").orElse(null)))
+                    throw new AssertionError("SlowDown response lacks Retry-After");
+                status(200, get(client, base, "/health", "192.0.2.2"));
+            } else if (configuration.containsKey("PUBLIC_BYTES_PER_SECOND")) {
+                long start = System.nanoTime();
+                byte[] upload = new byte[128];
+                status(200, client.send(HttpTest.signedUri(URI.create(base + "/objects/bandwidth"),
+                    "PUT", upload, Map.of()), HttpResponse.BodyHandlers.ofString()));
+                if (System.nanoTime() - start < 800_000_000L)
+                    throw new AssertionError("Upload bytes were not paced");
+                start = System.nanoTime();
+                status(200, get(client, base, "/health", "192.0.2.1"));
+                status(200, get(client, base, "/health", "192.0.2.1"));
+                if (System.nanoTime() - start < 250_000_000L)
+                    throw new AssertionError("Responses were not paced by the shared byte budget");
+            } else {
+                status(200, get(client, base, "/health", "192.0.2.1"));
+                status(503, get(client, base, "/health", "192.0.2.2"));
+            }
+        } finally {
+            server.stop(0);
+            executor.close();
+            store.close();
+            try (var paths = Files.walk(root)) {
+                for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path);
+            }
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        var disabled = ClientLimits.fromEnvironment(Map.of());
+        if (disabled == null) throw new AssertionError("Disabled configuration missing");
+        try { ClientLimits.fromEnvironment(Map.of("PUBLIC_TRUSTED_PROXY_IPS", "127.0.0.1"));
+            throw new AssertionError("Proxy trust accepted without limits");
+        } catch (IllegalArgumentException expected) { }
+        exercise(Map.of("PUBLIC_REQUESTS_PER_SECOND", "1", "PUBLIC_REQUEST_BURST", "1",
+            "PUBLIC_TRUSTED_PROXY_IPS", "127.0.0.1"), true);
+        exercise(Map.of("PUBLIC_REQUESTS_PER_SECOND", "1", "PUBLIC_REQUEST_BURST", "1"), false);
+        exercise(Map.of("PUBLIC_BYTES_PER_SECOND", "64", "PUBLIC_BYTE_BURST", "64"), false);
+        System.out.println("Client limit tests passed");
+    }
+}

test/cloud/lunarsky/store/ClusterIntegrationTest.java

@@ -3,8 +3,14 @@ package cloud.lunarsky.store;
 import java.io.ByteArrayInputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
 import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.UUID;
 
 public final class ClusterIntegrationTest {
     private static final String KEY = "cluster-test/survivor";
@@ -21,7 +27,9 @@ public final class ClusterIntegrationTest {
             switch (args[0]) {
                 case "basic" -> {
                     byte[] large = new byte[ClusterNode.MAX_SEGMENT + 37];
-                    for (int i = 0; i < large.length; i++) large[i] = (byte) (i * 31);
+                    for (int i = 0; i < large.length; i++) {
+                        large[i] = (byte) (i * 31);
+                    }
                     String largeKey = "cluster-test/large";
                     put(store, bucket, largeKey, large, false);
                     try (var opened = store.open(bucket, largeKey)) {
@@ -39,7 +47,10 @@ public final class ClusterIntegrationTest {
                             "Overwrite was not visible");
                     }
                     store.delete(bucket, largeKey);
-                    try { store.open(bucket, largeKey); throw new AssertionError("Deleted object remained visible"); }
+                    try {
+                        store.open(bucket, largeKey);
+                        throw new AssertionError("Deleted object remained visible");
+                    }
                     catch (StoreException error) { require(error.status == 404, "Wrong missing-object status"); }
                     put(store, bucket, KEY, stable, false);
                     require(store.ready(), "Healthy cluster is not ready");
@@ -57,13 +68,70 @@ public final class ClusterIntegrationTest {
                     }
                     System.out.println("Cluster degraded test passed");
                 }
+                case "multipart-stage" -> {
+                    String key = "cluster-test/multipart";
+                    String id = store.create(bucket, key, "text/plain");
+                    byte[] first = "hello ".getBytes(StandardCharsets.UTF_8);
+                    byte[] second = "world".getBytes(StandardCharsets.UTF_8);
+                    putPart(store, id, bucket, key, 1, "old".getBytes(StandardCharsets.UTF_8));
+                    putPart(store, id, bucket, key, 1, first);
+                    putPart(store, id, bucket, key, 2, second);
+                    require(store.activeUploads() == 1, "Upload was not retained");
+                    require(store.stagedBytes() == first.length + second.length, "Replaced part was counted twice");
+                    require(store.listUploads(bucket, key).size() == 1, "Upload listing missed the staged upload");
+                    var firstPage = store.listParts(id, bucket, key, 0, 1);
+                    require(firstPage.truncated() && firstPage.parts().size() == 1 && firstPage.nextMarker() == 1,
+                        "Part listing did not paginate");
+                    require(store.listParts(id, bucket, key, 1, 1).parts().getFirst().number() == 2,
+                        "Part marker skipped the second part");
+                    try {
+                        store.open(bucket, key);
+                        throw new AssertionError("Incomplete upload became visible");
+                    } catch (StoreException error) { require(error.status == 404, "Wrong incomplete-upload status"); }
+                    System.out.println("Cluster multipart parts staged and listed");
+                }
+                case "multipart-complete" -> {
+                    String key = "cluster-test/multipart";
+                    var uploads = store.listUploads(bucket, key);
+                    require(uploads.size() == 1, "Upload did not survive gateway restart");
+                    String id = uploads.getFirst().id();
+                    var listed = store.listParts(id, bucket, key, 0, 1000).parts();
+                    require(listed.size() == 2, "Staged parts were lost");
+                    try {
+                        store.complete(id, bucket, key, List.of(new MultipartStorage.Part(1, "0".repeat(32))));
+                        throw new AssertionError("Wrong part ETag was accepted");
+                    } catch (StoreException error) { require(error.status == 400, "Wrong ETag rejection status"); }
+                    var completed = store.complete(id, bucket, key, List.of(
+                        new MultipartStorage.Part(1, listed.get(0).etag()),
+                        new MultipartStorage.Part(2, listed.get(1).etag())));
+                    byte[] expected = "hello world".getBytes(StandardCharsets.UTF_8);
+                    require(completed.length() == expected.length && completed.contentType().equals("text/plain"),
+                        "Completed object metadata is wrong");
+                    MessageDigest digest = MessageDigest.getInstance("MD5");
+                    digest.update(HexFormat.of().parseHex(listed.get(0).etag()));
+                    digest.update(HexFormat.of().parseHex(listed.get(1).etag()));
+                    require(completed.etag().equals(HexFormat.of().formatHex(digest.digest()) + "-2"),
+                        "Multipart ETag is wrong");
+                    try (var opened = store.open(bucket, key)) {
+                        require(Arrays.equals(opened.stream().readAllBytes(), expected), "Completed multipart body is wrong");
+                    }
+                    require(store.activeUploads() == 0 && store.stagedBytes() == 0, "Completed parts still count as staged");
+                    String aborted = store.create(bucket, "cluster-test/aborted", "text/plain");
+                    putPart(store, aborted, bucket, "cluster-test/aborted", 1, expected);
+                    store.abort(aborted, bucket, "cluster-test/aborted");
+                    require(store.activeUploads() == 0 && store.stagedBytes() == 0, "Aborted parts still count as staged");
+                    System.out.println("Cluster multipart completion survived restart and node loss");
+                }
                 case "quorum-lost" -> {
                     require(!store.ready(), "One available node must not be ready");
                     try {
                         put(store, bucket, "cluster-test/rejected", new byte[]{1}, false);
                         throw new AssertionError("Write succeeded with only one node");
                     } catch (StoreException error) { require(error.status == 503, "Wrong unavailable status"); }
-                    try { store.open(bucket, "cluster-test/rejected"); throw new AssertionError("Failed write became visible"); }
+                    try {
+                        store.open(bucket, "cluster-test/rejected");
+                        throw new AssertionError("Failed write became visible");
+                    }
                     catch (StoreException error) { require(error.status == 404, "Partial object became visible"); }
                     System.out.println("Cluster quorum-loss test passed");
                 }
@@ -82,13 +150,18 @@ public final class ClusterIntegrationTest {
                     var executor = java.util.concurrent.Executors.newFixedThreadPool(2);
                     try {
                         var a = executor.submit(() -> {
-                            start.await(); put(store, bucket, key, first, false); return null;
+                            start.await();
+                            put(store, bucket, key, first, false);
+                            return null;
                         });
                         var b = executor.submit(() -> {
-                            start.await(); put(store, bucket, key, second, false); return null;
+                            start.await();
+                            put(store, bucket, key, second, false);
+                            return null;
                         });
                         start.countDown();
-                        a.get(); b.get();
+                        a.get();
+                        b.get();
                         try (var opened = store.open(bucket, key)) {
                             byte[] actual = opened.stream().readAllBytes();
                             require(Arrays.equals(actual, first) || Arrays.equals(actual, second),
@@ -130,6 +203,42 @@ public final class ClusterIntegrationTest {
                     }
                     System.out.println("Joined node accepted new placements while previous objects stayed readable");
                 }
+                case "verify-expanded" -> {
+                    for (int i = 0; i < 32; i++) {
+                        try (var opened = store.open(bucket, "cluster-test/expanded-" + i)) {
+                            require(("expanded object " + i).equals(new String(opened.stream().readAllBytes(),
+                                StandardCharsets.UTF_8)), "Expanded object was lost during maintenance");
+                        }
+                    }
+                    System.out.println("Expanded objects survived repair and cleanup");
+                }
+                case "balanced" -> {
+                    try (var connection = java.sql.DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
+                            env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"))) {
+                        NodeClient nodes = NodeRegistry.load(connection,
+                            Arrays.stream(urls).map(URI::create).toList(), env.get("CLUSTER_TOKEN"), null);
+                        int checked = 0;
+                        try (var query = connection.createStatement();
+                             var result = query.executeQuery("SELECT s.segment_id, s.replica_ids FROM cluster_segments s " +
+                                 "JOIN cluster_objects o ON o.generation=s.generation")) {
+                            while (result.next()) {
+                                UUID segment = (UUID) result.getObject(1);
+                                Set<UUID> preferred = new HashSet<>();
+                                Set<UUID> hosts = new HashSet<>();
+                                for (int index : PlacementPolicy.candidates(segment, nodes, true)) {
+                                    if (hosts.add(nodes.faultDomain(index, true))) preferred.add(nodes.node(index).id());
+                                    if (preferred.size() == 3) break;
+                                }
+                                Set<UUID> actual = new HashSet<>();
+                                for (Object id : (Object[]) result.getArray(2).getArray()) actual.add((UUID) id);
+                                require(actual.equals(preferred), "Segment did not move to preferred hosts");
+                                checked++;
+                            }
+                        }
+                        require(checked > 0, "No live segments were checked for placement");
+                    }
+                    System.out.println("Existing segments balanced across preferred hosts");
+                }
                 default -> throw new IllegalArgumentException("Unknown test phase");
             }
         }
@@ -138,6 +247,11 @@ public final class ClusterIntegrationTest {
         store.put(bucket, key, new ByteArrayInputStream(data), data.length, SigV4.hex(SigV4.hash(data)),
             null, createOnly, "application/octet-stream");
     }
+    private static void putPart(ClusterStore store, String id, String bucket, String key, int number, byte[] data)
+            throws Exception {
+        store.putPart(id, bucket, key, number, new ByteArrayInputStream(data), data.length,
+            SigV4.hex(SigV4.hash(data)), null);
+    }
     private static void require(boolean condition, String message) {
         if (!condition) throw new AssertionError(message);
     }

test/cloud/lunarsky/store/ClusterMetadataTest.java

@@ -0,0 +1,63 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpServer;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.DriverManager;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public final class ClusterMetadataTest {
+    public static void main(String[] args) throws Exception {
+        Map<String, String> env = System.getenv();
+        String token = "metadata-test-cluster-token-0123456789";
+        String repairToken = "metadata-test-repair-token-0123456789";
+        Path root = Files.createTempDirectory("objectstore-metadata-test-");
+        try (ClusterNode first = new ClusterNode(root.resolve("first"), token, repairToken, UUID.randomUUID());
+             ClusterNode second = new ClusterNode(root.resolve("second"), token, repairToken, UUID.randomUUID())) {
+            HttpServer firstServer = server(first);
+            HttpServer secondServer = server(second);
+            try {
+                List<URI> nodes = List.of(url(firstServer), url(secondServer));
+                try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"),
+                        env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"), "objects",
+                        nodes, token, repairToken, 1024 * 1024, 16 * 1024 * 1024, false)) {
+                    require(store.ready(), "Writable primary was not selected from the multi-host URL");
+                    try (var admin = DriverManager.getConnection(env.get("POSTGRES_ADMIN_JDBC_URL"),
+                            env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
+                         var statement = admin.createStatement()) {
+                        statement.execute("ALTER DATABASE objectstore_test SET default_transaction_read_only=on");
+                        try {
+                            require(!store.ready(), "Read-only metadata was reported ready");
+                        } finally {
+                            statement.execute("ALTER DATABASE objectstore_test RESET default_transaction_read_only");
+                        }
+                    }
+                    require(store.ready(), "Writable metadata did not recover after read-only mode ended");
+                }
+            } finally {
+                firstServer.stop(0);
+                secondServer.stop(0);
+            }
+        }
+        System.out.println("Metadata routing tests passed: second JDBC host and writable readiness");
+    }
+
+    private static HttpServer server(ClusterNode node) throws Exception {
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/", node::handle);
+        server.start();
+        return server;
+    }
+
+    private static URI url(HttpServer server) {
+        return URI.create("http://127.0.0.1:" + server.getAddress().getPort());
+    }
+
+    private static void require(boolean condition, String message) {
+        if (!condition) throw new AssertionError(message);
+    }
+}

test/cloud/lunarsky/store/ClusterMigrationTest.java

@@ -31,7 +31,9 @@ public final class ClusterMigrationTest {
         if (args[0].equals("create")) {
             UUID segment = UUID.randomUUID();
             byte[] hash = SigV4.hash(DATA);
-            for (int i = 0; i < nodes.count(); i++) nodes.put(i, segment, DATA, hash);
+            for (int i = 0; i < nodes.count(); i++) {
+                nodes.put(i, segment, DATA, hash);
+            }
             try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
                     env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
                  var statement = connection.createStatement()) {
@@ -40,18 +42,28 @@ public final class ClusterMigrationTest {
                 statement.execute("CREATE TABLE cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))");
                 statement.execute("CREATE TABLE cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))");
                 try (var insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, ?)")) {
-                    insert.setString(1, bucket); insert.setLong(2, DATA.length); insert.executeUpdate();
+                    insert.setString(1, bucket);
+                    insert.setLong(2, DATA.length);
+                    insert.executeUpdate();
                 }
                 UUID generation = UUID.randomUUID();
                 try (var insert = connection.prepareStatement("INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) {
-                    insert.setString(1, bucket); insert.setString(2, KEY); insert.setObject(3, generation);
+                    insert.setString(1, bucket);
-                    insert.setLong(4, DATA.length); insert.setLong(5, System.currentTimeMillis());
+                    insert.setString(2, KEY);
+                    insert.setObject(3, generation);
+                    insert.setLong(4, DATA.length);
+                    insert.setLong(5, System.currentTimeMillis());
                     insert.setString(6, HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(DATA)));
-                    insert.setBytes(7, hash); insert.setString(8, "text/plain"); insert.executeUpdate();
+                    insert.setBytes(7, hash);
+                    insert.setString(8, "text/plain");
+                    insert.executeUpdate();
                 }
                 try (var insert = connection.prepareStatement("INSERT INTO cluster_segments VALUES (?, 0, ?, ?, ?, '0,1,2')")) {
-                    insert.setObject(1, generation); insert.setObject(2, segment);
+                    insert.setObject(1, generation);
-                    insert.setInt(3, DATA.length); insert.setBytes(4, hash); insert.executeUpdate();
+                    insert.setObject(2, segment);
+                    insert.setInt(3, DATA.length);
+                    insert.setBytes(4, hash);
+                    insert.executeUpdate();
                 }
             }
             System.out.println("Legacy cluster fixture created");

test/cloud/lunarsky/store/ClusterNodeTest.java

@@ -75,6 +75,28 @@ public final class ClusterNodeTest {
                 client.repair(0, id, value, SigV4.hash(value));
                 require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))),
                     "Repair did not restore the original bytes");
+                UUID orphan = UUID.randomUUID();
+                client.put(0, orphan, value, SigV4.hash(value));
+                var inventory = client.inventory(0, orphan.toString().substring(0, 2), null);
+                var listed = inventory.stream().filter(entry -> entry.id().equals(orphan)).findFirst().orElseThrow();
+                var forgedInventory = HttpRequest.newBuilder(uri.resolve("/segments?shard=" +
+                    orphan.toString().substring(0, 2))).header("X-Cluster-Token", token)
+                    .header("X-Cluster-Expected-Node", nodeId.toString()).GET().build();
+                require(HttpClient.newHttpClient().send(forgedInventory, HttpResponse.BodyHandlers.discarding())
+                    .statusCode() == 403, "Gateway token was allowed to list node segments");
+                var forgedDelete = HttpRequest.newBuilder(uri.resolve("/segments/" + orphan))
+                    .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString())
+                    .header("X-Cluster-Expected-Mtime", Long.toString(listed.modified()))
+                    .header("X-Cluster-Gc-Min-Age-Millis", "0").DELETE().build();
+                require(HttpClient.newHttpClient().send(forgedDelete, HttpResponse.BodyHandlers.discarding())
+                    .statusCode() == 403, "Gateway token was allowed to delete a segment");
+                require(!client.deleteOrphan(0, new NodeClient.StoredSegment(orphan, listed.modified() - 1), 0),
+                    "Stale inventory entry deleted a segment");
+                require(client.deleteOrphan(0, listed, 0), "Confirmed orphan segment was not deleted");
+                try {
+                    client.get(0, orphan, value.length, SigV4.hash(value));
+                    throw new AssertionError("Deleted orphan was still readable");
+                } catch (IOException expected) { }
             } finally { server.stop(0); }
         }
         Path pending = root.resolve("pending").resolve("unfinished.part");
@@ -116,7 +138,25 @@ public final class ClusterNodeTest {
                 throw new AssertionError("Oversized segment response was accepted");
             } catch (IOException expected) { }
         } finally { oversized.stop(0); }
-        System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup");
+        URI stopped = URI.create("http://127.0.0.1:" + oversized.getAddress().getPort());
+        NodeClient readOnly = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, stopped)), token, null);
+        try {
+            readOnly.get(0, id, value.length, SigV4.hash(value));
+            throw new AssertionError("Read-only access did not detect a stopped node");
+        } catch (IOException expected) {
+            require(expected.getMessage().contains("temporarily unreachable"),
+                "Read-only access did not use the short health probe");
+        }
+        NodeClient offline = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, stopped)), token, null);
+        require(!offline.availableHostsAtLeast(1, false), "Stopped node was reported healthy");
+        try {
+            offline.get(0, id, value.length, SigV4.hash(value));
+            throw new AssertionError("Stopped node was read after a failed health check");
+        } catch (IOException expected) {
+            require(expected.getMessage().contains("temporarily unreachable"),
+                "Read did not skip a recently failed node");
+        }
+        System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup, outage fallback");
     }
     private static void require(boolean condition, String message) {
         if (!condition) throw new AssertionError(message);

test/cloud/lunarsky/store/ClusterTlsTest.java

@@ -0,0 +1,107 @@
+package cloud.lunarsky.store;
+
+import com.sun.net.httpserver.HttpServer;
+import com.sun.net.httpserver.HttpsServer;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public final class ClusterTlsTest {
+    private static final String PASSWORD = "local-test-password-0123456789";
+
+    public static void main(String[] args) throws Exception {
+        Path directory = Files.createTempDirectory("objectstore-tls-");
+        Path keyStore = directory.resolve("node.p12");
+        Path trustStore = directory.resolve("trust.p12");
+        Path certificate = directory.resolve("node.crt");
+        Path passwordFile = directory.resolve("password");
+        Files.writeString(passwordFile, PASSWORD + "\n");
+        String keytool = Path.of(System.getProperty("java.home"), "bin", "keytool").toString();
+        run(keytool, "-genkeypair", "-alias", "node", "-keyalg", "RSA", "-keysize", "2048",
+            "-validity", "2", "-dname", "CN=localhost", "-ext", "SAN=DNS:localhost",
+            "-storetype", "PKCS12", "-keystore", keyStore.toString(), "-storepass", PASSWORD,
+            "-keypass", PASSWORD, "-noprompt");
+        run(keytool, "-exportcert", "-alias", "node", "-keystore", keyStore.toString(),
+            "-storepass", PASSWORD, "-file", certificate.toString());
+        run(keytool, "-importcert", "-alias", "node", "-file", certificate.toString(),
+            "-keystore", trustStore.toString(), "-storetype", "PKCS12", "-storepass", PASSWORD,
+            "-noprompt");
+
+        Map<String, String> serverConfig = Map.of(
+            "NODE_TLS_KEYSTORE", keyStore.toString(), "NODE_TLS_PASSWORD_FILE", passwordFile.toString());
+        Map<String, String> clientConfig = Map.of(
+            "CLUSTER_TLS_TRUSTSTORE", trustStore.toString(),
+            "CLUSTER_TLS_PASSWORD_FILE", passwordFile.toString());
+        String token = "tls-test-cluster-token-0123456789";
+        String repairToken = "tls-test-repair-token-0123456789";
+        UUID hostId = UUID.randomUUID();
+        byte[] data = "encrypted transport".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        try (ClusterNode node = new ClusterNode(directory.resolve("data"), token, repairToken, hostId)) {
+            HttpServer server = ClusterTls.nodeServer(new InetSocketAddress("127.0.0.1", 0), serverConfig);
+            require(server instanceof HttpsServer, "Node did not enable HTTPS");
+            server.createContext("/", node::handle);
+            server.start();
+            try {
+                URI url = URI.create("https://localhost:" + server.getAddress().getPort());
+                HttpClient trusted = ClusterTls.client(clientConfig, Duration.ofSeconds(3));
+                NodeIdentity identity = NodeClient.probe(url, token, trusted);
+                require(identity.hostId().equals(hostId), "TLS probe returned wrong node identity");
+                NodeClient client = new NodeClient(List.of(
+                    new NodeClient.Node(identity.nodeId(), hostId, url)), token, repairToken, trusted);
+                UUID segment = UUID.randomUUID();
+                client.put(0, segment, data, SigV4.hash(data));
+                require(java.util.Arrays.equals(data, client.get(0, segment, data.length, SigV4.hash(data))),
+                    "TLS segment roundtrip failed");
+                HttpRequest request = HttpRequest.newBuilder(url.resolve("/identity"))
+                    .header("X-Cluster-Token", token).GET().build();
+                try {
+                    ClusterTls.client(Map.of(), Duration.ofSeconds(3))
+                        .send(request, HttpResponse.BodyHandlers.discarding());
+                    throw new AssertionError("Untrusted certificate was accepted");
+                } catch (IOException expected) { }
+                URI wrongHost = URI.create("https://127.0.0.1:" + server.getAddress().getPort());
+                try {
+                    NodeClient.probe(wrongHost, token, trusted);
+                    throw new AssertionError("Wrong certificate hostname was accepted");
+                } catch (IOException expected) { }
+            } finally {
+                server.stop(0);
+            }
+        }
+        try {
+            ClusterTls.nodeServer(new InetSocketAddress("127.0.0.1", 0),
+                Map.of("NODE_TLS_KEYSTORE", keyStore.toString()));
+            throw new AssertionError("Incomplete TLS configuration was accepted");
+        } catch (IOException expected) { }
+        try {
+            ClusterTls.client(Map.of("CLUSTER_TLS_TRUSTSTORE", trustStore.toString()),
+                Duration.ofSeconds(3));
+            throw new AssertionError("Incomplete cluster trust configuration was accepted");
+        } catch (IOException expected) { }
+        try {
+            NodeClient.validateUrl(URI.create("http://localhost:9100"), true);
+            throw new AssertionError("HTTP node URL was accepted with cluster TLS enabled");
+        } catch (IllegalArgumentException expected) { }
+        System.out.println("Cluster TLS tests passed: trusted roundtrip, untrusted and hostname rejection, no HTTP downgrade");
+    }
+
+    private static void run(String... command) throws Exception {
+        Process process = new ProcessBuilder(command).redirectErrorStream(true).start();
+        String output = new String(process.getInputStream().readAllBytes(),
+            java.nio.charset.StandardCharsets.UTF_8);
+        if (process.waitFor() != 0) throw new AssertionError("keytool failed: " + output);
+    }
+
+    private static void require(boolean condition, String message) {
+        if (!condition) throw new AssertionError(message);
+    }
+}

test/cloud/lunarsky/store/EncryptedVolumeTest.java

@@ -0,0 +1,42 @@
+package cloud.lunarsky.store;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+public final class EncryptedVolumeTest {
+    private static void rejected(Map<String, String> configuration) throws Exception {
+        try {
+            EncryptedVolume.requireConfigured(configuration);
+            throw new AssertionError("Unavailable encrypted storage was accepted");
+        } catch (IOException expected) { }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Path directory = Files.createTempDirectory("objectstore-encrypted-volume-");
+        Path marker = directory.resolve(".objectstore-encrypted");
+        String id = "0123456789abcdef0123456789abcdef";
+        Map<String, String> configuration = Map.of(
+            "ENCRYPTED_VOLUME_MARKER_FILE", marker.toString(), "ENCRYPTED_VOLUME_ID", id);
+        try {
+            EncryptedVolume.requireConfigured(Map.of());
+            rejected(configuration);
+            Files.writeString(marker, id + "\n");
+            EncryptedVolume.requireConfigured(configuration);
+            rejected(Map.of("ENCRYPTED_VOLUME_MARKER_FILE", marker.toString()));
+            rejected(Map.of("ENCRYPTED_VOLUME_MARKER_FILE", marker.toString(),
+                "ENCRYPTED_VOLUME_ID", "fedcba9876543210fedcba9876543210"));
+            Files.delete(marker);
+            Path elsewhere = directory.resolve("elsewhere");
+            Files.writeString(elsewhere, id + "\n");
+            Files.createSymbolicLink(marker, elsewhere);
+            rejected(configuration);
+        } finally {
+            Files.deleteIfExists(marker);
+            Files.deleteIfExists(directory.resolve("elsewhere"));
+            Files.delete(directory);
+        }
+        System.out.println("Encrypted volume marker tests passed");
+    }
+}

test/cloud/lunarsky/store/HttpTest.java

@@ -13,14 +13,20 @@ import java.time.Clock;
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
+import java.util.Base64;
 import java.util.Comparator;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.Executors;
+import java.util.zip.CRC32;
+import java.util.zip.CRC32C;
+import java.util.zip.Checksum;
 
 public final class HttpTest {
     private static final String ACCESS = "TESTACCESSKEY123";
     private static final String SECRET = "test-secret-key-that-is-at-least-32-characters";
+    private static final String SECONDARY = "SECONDARYKEY1234";
+    private static final String SECONDARY_SECRET = "secondary-secret-key-that-is-at-least-32-characters";
     private static final String REGION = "us-east-1";
     private static final DateTimeFormatter DATE =
         DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);
@@ -29,7 +35,12 @@ public final class HttpTest {
         return signedUri(URI.create(base + "/objects/" + SigV4.encode(key, true)), method, body, Map.of());
     }
 
-    private static HttpRequest signedUri(URI uri, String method, byte[] body, Map<String, String> extra) {
+    static HttpRequest signedUri(URI uri, String method, byte[] body, Map<String, String> extra) {
+        return signedUriAs(uri, method, body, extra, ACCESS, SECRET);
+    }
+
+    private static HttpRequest signedUriAs(URI uri, String method, byte[] body, Map<String, String> extra,
+                                           String access, String secret) {
         String host = uri.getAuthority();
         String date = DATE.format(Instant.now());
         String hash = SigV4.hex(SigV4.hash(body));
@@ -46,11 +57,11 @@ public final class HttpTest {
         String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
             + SigV4.hex(SigV4.hash(canonical.toString().getBytes(StandardCharsets.UTF_8)));
         String signature = SigV4.hex(SigV4.hmac(
-            SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign));
+            SigV4.signingKey(secret, date.substring(0, 8), REGION), toSign));
         HttpRequest.Builder request = HttpRequest.newBuilder(uri)
             .header("x-amz-date", date)
             .header("x-amz-content-sha256", hash)
-            .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/"
+            .header("authorization", "AWS4-HMAC-SHA256 Credential=" + access + "/"
                 + scope + ",SignedHeaders=" + names + ",Signature=" + signature);
         extra.forEach(request::header);
         return request.method(method, body.length == 0
@@ -59,26 +70,279 @@ public final class HttpTest {
             .build();
     }
 
-    private static void status(int expected, HttpResponse<byte[]> response) {
+    private static URI presignedUri(URI uri, String method, int expires) {
+        String date = DATE.format(Instant.now());
+        String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
+        String query = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=" +
+            SigV4.encode(ACCESS + "/" + scope, false) + "&X-Amz-Date=" + date +
+            "&X-Amz-Expires=" + expires + "&X-Amz-SignedHeaders=host";
+        String canonical = method + "\n" + uri.getRawPath() + "\n" +
+            SigV4.canonicalQuery(query) + "\nhost:" + uri.getAuthority() +
+            "\n\nhost\nUNSIGNED-PAYLOAD";
+        String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" +
+            SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)));
+        String signature = SigV4.hex(SigV4.hmac(SigV4.signingKey(SECRET,
+            date.substring(0, 8), REGION), toSign));
+        return URI.create(uri + "?" + query + "&X-Amz-Signature=" + signature);
+    }
+
+    private static void testPresigned(HttpClient client, String base) throws Exception {
+        URI object = URI.create(base + "/objects/presigned-test");
+        byte[] body = "presigned upload".getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(HttpRequest.newBuilder(presignedUri(object, "PUT", 60))
+            .PUT(HttpRequest.BodyPublishers.ofByteArray(body)).build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var read = client.send(HttpRequest.newBuilder(presignedUri(object, "GET", 60)).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, read);
+        if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Presigned object mismatch");
+        URI tampered = URI.create(presignedUri(object, "GET", 60).toString().replace("presigned-test", "different"));
+        status(403, client.send(HttpRequest.newBuilder(tampered).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(HttpRequest.newBuilder(presignedUri(object, "DELETE", 60))
+            .DELETE().build(), HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testAcl(HttpClient client, String base) throws Exception {
+        byte[] body = "private object".getBytes(StandardCharsets.UTF_8);
+        URI object = URI.create(base + "/objects/acl-test");
+        URI objectAcl = URI.create(object + "?acl");
+        status(200, client.send(signedUri(object, "PUT", body, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(),
+            SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(),
+            SECONDARY, SECRET), HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(signedUriAs(URI.create(base + "/_objectstore/capabilities"),
+            "GET", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(HttpRequest.newBuilder(object).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(objectAcl, "PUT", new byte[0],
+            Map.of("x-amz-grant-read", "id=\"" + SECONDARY + "\"")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var read = client.send(signedUriAs(object, "GET", new byte[0], Map.of(),
+            SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray());
+        status(200, read);
+        if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("ACL read mismatch");
+        status(403, client.send(signedUriAs(object, "DELETE", new byte[0], Map.of(),
+            SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
+        var acl = client.send(signedUri(objectAcl, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        status(200, acl);
+        if (!acl.body().contains(SECONDARY)) throw new AssertionError("Object ACL grant missing");
+        status(200, client.send(signedUri(URI.create(object + "?acl&x-id=GetObjectAcl"),
+            "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(objectAcl, "PUT",
+            acl.body().getBytes(StandardCharsets.UTF_8), Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+
+        URI bucketAcl = URI.create(base + "/objects?acl");
+        status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0],
+            Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(HttpRequest.newBuilder(
+            URI.create(base + "/objects?list-type=2")).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(HttpRequest.newBuilder(
+            URI.create(base + "/objects?uploads")).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0],
+            Map.of("x-amz-grant-write", "id=\"" + SECONDARY + "\"")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        URI uploaded = URI.create(base + "/objects/secondary-upload");
+        status(200, client.send(signedUriAs(uploaded, "PUT", body, Map.of(),
+            SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(signedUriAs(uploaded, "GET", new byte[0], Map.of(),
+            SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(uploaded, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(object, "PUT", body,
+            Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(HttpRequest.newBuilder(object).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(HttpRequest.newBuilder(object).DELETE().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+
+        URI historyBucket = URI.create(base + "/acl-history");
+        status(200, client.send(signedUri(historyBucket, "PUT", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        byte[] versioning = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"
+            .getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(signedUri(URI.create(historyBucket + "?versioning"),
+            "PUT", versioning, Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+        URI versioned = URI.create(historyBucket + "/versioned");
+        var first = client.send(signedUri(versioned, "PUT", body, Map.of("x-amz-acl", "public-read")),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, first);
+        String oldVersion = first.headers().firstValue("x-amz-version-id").orElseThrow();
+        status(200, client.send(signedUri(versioned, "PUT", body, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(HttpRequest.newBuilder(versioned).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        URI historical = URI.create(versioned + "?versionId=" + oldVersion);
+        status(200, client.send(HttpRequest.newBuilder(historical).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        URI historicalAcl = URI.create(historical + "&acl");
+        status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0],
+            Map.of("x-amz-grant-write-acp", "id=\"" + SECONDARY + "\"")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUriAs(historicalAcl, "PUT", new byte[0],
+            Map.of("x-amz-acl", "public-read"), SECONDARY, SECONDARY_SECRET),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(HttpRequest.newBuilder(historical).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0],
+            Map.of("x-amz-acl", "private")), HttpResponse.BodyHandlers.ofByteArray()));
+        status(403, client.send(HttpRequest.newBuilder(historical).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testStreaming(HttpClient client, String base) throws Exception {
+        URI object = URI.create(base + "/objects/streaming-test");
+        byte[] body = "verified streaming payload".getBytes(StandardCharsets.UTF_8);
+        String date = DATE.format(Instant.now());
+        String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
+        String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+        String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length";
+        String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" +
+            "host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode +
+            "\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length +
+            "\n\n" + names + "\n" + mode;
+        byte[] signingKey = SigV4.signingKey(SECRET, date.substring(0, 8), REGION);
+        String seed = SigV4.hex(SigV4.hmac(signingKey, "AWS4-HMAC-SHA256\n" + date + "\n" +
+            scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)))));
+        String previous = seed;
+        var encoded = new java.io.ByteArrayOutputStream();
+        for (byte[] chunk : new byte[][]{body, new byte[0]}) {
+            String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" +
+                previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" +
+                SigV4.hex(SigV4.hash(chunk));
+            previous = SigV4.hex(SigV4.hmac(signingKey, toSign));
+            encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n")
+                .getBytes(StandardCharsets.US_ASCII));
+            encoded.write(chunk);
+            encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII));
+        }
+        HttpRequest.Builder request = HttpRequest.newBuilder(object)
+            .header("content-encoding", "aws-chunked")
+            .header("x-amz-content-sha256", mode)
+            .header("x-amz-date", date)
+            .header("x-amz-decoded-content-length", Integer.toString(body.length))
+            .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope +
+                ",SignedHeaders=" + names + ",Signature=" + seed);
+        status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(encoded.toByteArray()))
+            .build(), HttpResponse.BodyHandlers.ofByteArray()));
+        var read = client.send(signedUri(object, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, read);
+        if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Streaming upload mismatch");
+        status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testStreamingTrailer(HttpClient client, String base) throws Exception {
+        URI object = URI.create(base + "/objects/streaming-trailer-test");
+        byte[] body = "verified trailer payload".getBytes(StandardCharsets.UTF_8);
+        String date = DATE.format(Instant.now());
+        String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
+        String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
+        String trailerName = "x-amz-checksum-xxhash3";
+        String checksum = encodedChecksum("XXHASH3", body);
+        String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;" +
+            "x-amz-decoded-content-length;x-amz-sdk-checksum-algorithm;x-amz-trailer";
+        String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" +
+            "host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode +
+            "\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length +
+            "\nx-amz-sdk-checksum-algorithm:XXHASH3\nx-amz-trailer:" + trailerName +
+            "\n\n" + names + "\n" + mode;
+        byte[] key = SigV4.signingKey(SECRET, date.substring(0, 8), REGION);
+        String seed = SigV4.hex(SigV4.hmac(key, "AWS4-HMAC-SHA256\n" + date + "\n" +
+            scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)))));
+        String previous = seed;
+        var encoded = new java.io.ByteArrayOutputStream();
+        for (byte[] chunk : new byte[][]{body, new byte[0]}) {
+            String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" +
+                previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" +
+                SigV4.hex(SigV4.hash(chunk));
+            previous = SigV4.hex(SigV4.hmac(key, toSign));
+            encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n")
+                .getBytes(StandardCharsets.US_ASCII));
+            encoded.write(chunk);
+            if (chunk.length != 0) encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII));
+        }
+        encoded.write((trailerName + ":" + checksum + "\r\n").getBytes(StandardCharsets.US_ASCII));
+        String trailerToSign = "AWS4-HMAC-SHA256-TRAILER\n" + date + "\n" + scope + "\n" +
+            previous + "\n" + SigV4.hex(SigV4.hash((trailerName + ":" + checksum + "\n")
+                .getBytes(StandardCharsets.UTF_8)));
+        encoded.write(("x-amz-trailer-signature=" + SigV4.hex(SigV4.hmac(key, trailerToSign)) +
+            "\r\n\r\n").getBytes(StandardCharsets.US_ASCII));
+        byte[] upload = encoded.toByteArray();
+        HttpRequest.Builder request = HttpRequest.newBuilder(object)
+            .header("content-encoding", "aws-chunked")
+            .header("x-amz-content-sha256", mode)
+            .header("x-amz-date", date)
+            .header("x-amz-decoded-content-length", Integer.toString(body.length))
+            .header("x-amz-sdk-checksum-algorithm", "XXHASH3")
+            .header("x-amz-trailer", trailerName)
+            .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope +
+                ",SignedHeaders=" + names + ",Signature=" + seed);
+        status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(upload)).build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var head = client.send(signedUri(object, "HEAD", new byte[0],
+            Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding());
+        status(200, head);
+        if (!checksum.equals(head.headers().firstValue(trailerName).orElse("")))
+            throw new AssertionError("Signed checksum trailer was not persisted");
+        byte[] tampered = upload.clone();
+        tampered[upload.length - 10] ^= 1;
+        status(400, client.send(HttpRequest.newBuilder(object)
+            .header("content-encoding", "aws-chunked")
+            .header("x-amz-content-sha256", mode)
+            .header("x-amz-date", date)
+            .header("x-amz-decoded-content-length", Integer.toString(body.length))
+            .header("x-amz-sdk-checksum-algorithm", "XXHASH3")
+            .header("x-amz-trailer", trailerName)
+            .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope +
+                ",SignedHeaders=" + names + ",Signature=" + seed)
+            .PUT(HttpRequest.BodyPublishers.ofByteArray(tampered)).build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testCapabilities(HttpClient client, String base) throws Exception {
+        URI uri = URI.create(base + "/_objectstore/capabilities");
+        status(403, client.send(HttpRequest.newBuilder(uri).GET().build(),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var response = client.send(signedUri(uri, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        status(200, response);
+        String compact = response.body().replaceAll("\\s+", "");
+        if (!compact.contains("\"schemaVersion\":1") ||
+            !compact.contains("\"service\":\"lunarsky-objectstore\"") ||
+            !compact.contains("\"serviceVersion\":\"" + Version.VALUE + "\"") ||
+            !compact.contains("\"storageMode\":\"disk\"") ||
+            !compact.contains("\"ListParts\"") ||
+            !compact.contains("\"maxObjectBytes\":1024") ||
+            !compact.contains("\"maxTotalBytes\":4096") ||
+            !compact.contains("\"maxParts\":10000"))
+            throw new AssertionError("Unexpected capability manifest: " + response.body());
+        if (!response.headers().firstValue("content-type").orElse("").startsWith("application/json") ||
+            !"no-store".equals(response.headers().firstValue("cache-control").orElse("")))
+            throw new AssertionError("Capability response headers missing");
+        status(400, client.send(signedUri(URI.create(uri + "?extra=1"), "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void status(int expected, HttpResponse<?> response) {
         if (response.statusCode() != expected) {
             throw new AssertionError("Expected HTTP " + expected + ", got " + response.statusCode()
-                + ": " + new String(response.body(), StandardCharsets.UTF_8));
+                + ": " + (response.body() instanceof byte[] bytes
+                    ? new String(bytes, StandardCharsets.UTF_8) : response.body()));
         }
     }
 
-    public static void main(String[] args) throws Exception {
+    private static void testObjects(HttpClient client, String base) throws Exception {
-        Path root = Files.createTempDirectory("store-http-test-");
-        var executor = Executors.newVirtualThreadPerTaskExecutor();
-        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16);
-        DiskStore store = new DiskStore(root, 1024, 4096);
-        try {
-            var app = new Main(store,
-                new SigV4(ACCESS, SECRET, REGION, Clock.systemUTC()), "objects");
-            server.setExecutor(executor);
-            server.createContext("/", app::handle);
-            server.start();
-            String base = "http://127.0.0.1:" + server.getAddress().getPort();
-            HttpClient client = HttpClient.newHttpClient();
         status(200, client.send(HttpRequest.newBuilder(URI.create(base + "/health")).GET().build(),
             HttpResponse.BodyHandlers.ofByteArray()));
         String key = "folder/moon-☾.txt";
@@ -109,6 +373,9 @@ public final class HttpTest {
             throw new AssertionError("Range response mismatch");
         status(416, client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
             new byte[0], Map.of("range", "bytes=20-30")), HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testListing(HttpClient client, String base) throws Exception {
         var listed = client.send(signedUri(URI.create(base + "/objects?list-type=2&prefix=folder%2F"),
             "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
         if (listed.statusCode() != 200 || !listed.body().contains("<Key>folder/stars.txt</Key>") ||
@@ -136,6 +403,139 @@ public final class HttpTest {
             "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
         if (emptyPage.statusCode() != 200 || !emptyPage.body().contains("<KeyCount>0</KeyCount>"))
             throw new AssertionError("Empty list page failed: " + emptyPage.body());
+    }
+
+    private static void testCopy(HttpClient client, String base) throws Exception {
+        String source = "folder/copy source.txt";
+        String target = "folder/copied.txt";
+        byte[] body = "copy source".getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(signedUri(URI.create(base + "/objects/" + SigV4.encode(source, true)),
+            "PUT", body, Map.of("content-type", "text/plain")), HttpResponse.BodyHandlers.ofByteArray()));
+        String header = "/objects/" + SigV4.encode(source, true);
+        var result = client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", header)), HttpResponse.BodyHandlers.ofString());
+        if (result.statusCode() != 200 || !result.body().contains("<CopyObjectResult>") ||
+            !result.body().contains("<ETag>&quot;"))
+            throw new AssertionError("CopyObject failed: " + result.body());
+        var copied = client.send(signed(base, "GET", target, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
+        status(200, copied);
+        if (!java.util.Arrays.equals(body, copied.body()) ||
+            !"text/plain".equals(copied.headers().firstValue("content-type").orElse("")))
+            throw new AssertionError("Copied body or content type mismatch");
+        status(200, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", header, "x-amz-metadata-directive", "REPLACE",
+                "content-type", "text/markdown")), HttpResponse.BodyHandlers.ofByteArray()));
+        var replaced = client.send(signed(base, "GET", target, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
+        status(200, replaced);
+        if (!java.util.Arrays.equals(body, replaced.body()) ||
+            !"text/markdown".equals(replaced.headers().firstValue("content-type").orElse("")))
+            throw new AssertionError("REPLACE metadata directive failed");
+        status(200, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", "/objects/" + target)),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", "/objects/missing")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", "/other/source")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", "/objects/source?versionId=1")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(501, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", header, "x-amz-metadata-directive", "REPLACE",
+                "content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(signed(base, "DELETE", source, new byte[0]),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(signed(base, "DELETE", target, new byte[0]),
+            HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static String encodedChecksum(String algorithm, byte[] body) throws Exception {
+        if (algorithm.startsWith("XXHASH")) {
+            var checksum = new XxHashes(algorithm);
+            checksum.update(body, 0, body.length);
+            return Base64.getEncoder().encodeToString(checksum.digest());
+        }
+        if (algorithm.startsWith("CRC")) {
+            Checksum checksum = switch (algorithm) {
+                case "CRC32" -> new CRC32();
+                case "CRC32C" -> new CRC32C();
+                case "CRC64NVME" -> new Crc64Nvme();
+                default -> throw new IllegalArgumentException(algorithm);
+            };
+            checksum.update(body, 0, body.length);
+            long value = checksum.getValue();
+            byte[] bytes = new byte[algorithm.equals("CRC64NVME") ? 8 : 4];
+            for (int i = bytes.length - 1; i >= 0; i--) {
+                bytes[i] = (byte) value;
+                value >>>= 8;
+            }
+            return Base64.getEncoder().encodeToString(bytes);
+        }
+        String name = algorithm.equals("SHA1") ? "SHA-1" :
+            algorithm.equals("SHA256") ? "SHA-256" :
+            algorithm.equals("SHA512") ? "SHA-512" : "MD5";
+        return Base64.getEncoder().encodeToString(java.security.MessageDigest.getInstance(name).digest(body));
+    }
+
+    private static void testChecksums(HttpClient client, String base) throws Exception {
+        URI uri = URI.create(base + "/objects/checksum-target");
+        byte[] body = "checksum payload".getBytes(StandardCharsets.UTF_8);
+        String md5 = encodedChecksum("MD5", body);
+        for (String algorithm : new String[]{"CRC32", "CRC32C", "CRC64NVME", "XXHASH64",
+            "XXHASH3", "XXHASH128", "SHA1", "SHA256", "SHA512", "MD5"}) {
+            String header = "x-amz-checksum-" + algorithm.toLowerCase(java.util.Locale.ROOT);
+            String checksum = encodedChecksum(algorithm, body);
+            var stored = client.send(signedUri(uri, "PUT", body,
+                Map.of("content-md5", md5, header, checksum, "x-amz-sdk-checksum-algorithm", algorithm)),
+                HttpResponse.BodyHandlers.ofByteArray());
+            status(200, stored);
+            if (!checksum.equals(stored.headers().firstValue(header).orElse("")))
+                throw new AssertionError("Missing checksum response: " + algorithm);
+            var bad = client.send(signedUri(uri, "PUT", body,
+                Map.of(header, Base64.getEncoder().encodeToString(new byte[algorithm.equals("XXHASH128") ? 16 :
+                    algorithm.startsWith("XXHASH") || algorithm.equals("CRC64NVME") ? 8 :
+                    algorithm.startsWith("CRC") ? 4 :
+                    algorithm.equals("SHA1") ? 20 : algorithm.equals("SHA256") ? 32 :
+                    algorithm.equals("SHA512") ? 64 : 16]))), HttpResponse.BodyHandlers.ofString());
+            if (bad.statusCode() != 400 || !bad.body().contains("BadDigest"))
+                throw new AssertionError("Mismatched " + algorithm + " accepted: " + bad.body());
+            var unchanged = client.send(signedUri(uri, "GET", new byte[0], Map.of()),
+                HttpResponse.BodyHandlers.ofByteArray());
+            status(200, unchanged);
+            if (!java.util.Arrays.equals(body, unchanged.body()))
+                throw new AssertionError("Bad checksum replaced stored object");
+            var head = client.send(signedUri(uri, "HEAD", new byte[0],
+                Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding());
+            status(200, head);
+            if (!checksum.equals(head.headers().firstValue(header).orElse("")))
+                throw new AssertionError("Checksum was not persisted: " + algorithm);
+            status(400, client.send(signedUri(uri, "HEAD", new byte[0],
+                Map.of("x-amz-checksum-mode", "INVALID")), HttpResponse.BodyHandlers.discarding()));
+        }
+        var badMd5 = client.send(signedUri(uri, "PUT", body,
+            Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString());
+        if (badMd5.statusCode() != 400 || !badMd5.body().contains("BadDigest"))
+            throw new AssertionError("Mismatched Content-MD5 accepted");
+        status(400, client.send(signedUri(uri, "PUT", body,
+            Map.of("content-md5", "invalid")), HttpResponse.BodyHandlers.ofByteArray()));
+        status(400, client.send(signedUri(uri, "PUT", body,
+            Map.of("x-amz-checksum-crc32", encodedChecksum("CRC32", body),
+                "x-amz-sdk-checksum-algorithm", "CRC32C")), HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(uri, "PUT", body, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var automatic = client.send(signedUri(uri, "HEAD", new byte[0],
+            Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding());
+        status(200, automatic);
+        if (!encodedChecksum("CRC64NVME", body).equals(automatic.headers()
+            .firstValue("x-amz-checksum-crc64nvme").orElse("")))
+            throw new AssertionError("Default CRC64NVME checksum was not persisted");
+        status(204, client.send(signedUri(uri, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testMultipart(HttpClient client, String base) throws Exception {
         String movie = "folder/video.mp4";
         URI initiate = URI.create(base + "/objects/" + movie + "?uploads=");
         var created = client.send(signedUri(initiate, "POST", new byte[0],
@@ -148,7 +548,23 @@ public final class HttpTest {
             "?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
         var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie +
             "?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
-            status(200, partOne); status(200, partTwo);
+        status(200, partOne);
+        status(200, partTwo);
+        var rejectedPart = client.send(signedUri(URI.create(base + "/objects/" + movie +
+            "?partNumber=1&uploadId=" + upload), "PUT", first,
+            Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString());
+        if (rejectedPart.statusCode() != 400 || !rejectedPart.body().contains("BadDigest"))
+            throw new AssertionError("Mismatched part Content-MD5 accepted");
+        var parts = client.send(signedUri(URI.create(base + "/objects/" + movie +
+            "?uploadId=" + upload + "&max-parts=1"), "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        if (parts.statusCode() != 200 || !parts.body().contains("<IsTruncated>true</IsTruncated>") ||
+            !parts.body().contains("<PartNumber>1</PartNumber>"))
+            throw new AssertionError("Multipart part listing failed: " + parts.body());
+        var uploads = client.send(signedUri(URI.create(base + "/objects?uploads&prefix=folder%2F"),
+            "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+        if (uploads.statusCode() != 200 || !uploads.body().contains("<UploadId>" + upload + "</UploadId>"))
+            throw new AssertionError("Multipart upload listing failed: " + uploads.body());
         String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
             partOne.headers().firstValue("etag").orElseThrow() +
             "</ETag></Part><Part><PartNumber>2</PartNumber><ETag>" +
@@ -161,11 +577,17 @@ public final class HttpTest {
         if (!"hello world".equals(new String(assembled.body(), StandardCharsets.UTF_8)) ||
             !"video/mp4".equals(assembled.headers().firstValue("content-type").orElse("")))
             throw new AssertionError("Completed multipart object mismatch");
+        status(404, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload),
+            "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
         var abandoned = client.send(signedUri(URI.create(base + "/objects/abandoned?uploads="),
             "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
         String abandonedId = abandoned.body().split("<UploadId>")[1].split("</UploadId>")[0];
         status(204, client.send(signedUri(URI.create(base + "/objects/abandoned?uploadId=" + abandonedId),
             "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testDelete(HttpClient client, String base) throws Exception {
+        String key = "folder/moon-☾.txt";
         var head = client.send(signed(base, "HEAD", key, new byte[0]),
             HttpResponse.BodyHandlers.ofByteArray());
         status(200, head);
@@ -174,7 +596,311 @@ public final class HttpTest {
             HttpResponse.BodyHandlers.ofByteArray()));
         status(404, client.send(signed(base, "GET", key, new byte[0]),
             HttpResponse.BodyHandlers.ofByteArray()));
-            System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE, MIME, ranges, listing, multipart");
+    }
+
+    private static void testAttributes(HttpClient client, String base) throws Exception {
+        URI object = URI.create(base + "/objects/attributes.txt");
+        byte[] body = "object attributes".getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(signedUri(object, "PUT", body, Map.of(
+            "x-amz-meta-project", "LunarSky", "x-amz-tagging", "kind=test&phase=one")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var get = client.send(signedUri(object, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, get);
+        if (!"LunarSky".equals(get.headers().firstValue("x-amz-meta-project").orElse("")) ||
+            !"2".equals(get.headers().firstValue("x-amz-tagging-count").orElse("")))
+            throw new AssertionError("Stored attributes missing from GET");
+        var tagging = URI.create(object + "?tagging");
+        var originalTags = client.send(signedUri(URI.create(tagging + "&x-id=GetObjectTagging"),
+            "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        status(200, originalTags);
+        if (!originalTags.body().contains("<Key>kind</Key><Value>test</Value>"))
+            throw new AssertionError("Object tags were not stored");
+        byte[] replacement = "<Tagging><TagSet><Tag><Key>stage</Key><Value>two</Value></Tag></TagSet></Tagging>"
+            .getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(signedUri(tagging, "PUT", replacement, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(400, client.send(signedUri(tagging, "PUT", replacement,
+            Map.of("content-md5", Base64.getEncoder().encodeToString(new byte[16]))),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var replaced = client.send(signedUri(tagging, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        if (!replaced.body().contains("<Key>stage</Key><Value>two</Value>") ||
+            replaced.body().contains("<Key>kind</Key>")) throw new AssertionError("Tag replacement failed");
+        status(400, client.send(signedUri(tagging, "PUT", "<!DOCTYPE x [<!ENTITY y SYSTEM 'file:///etc/passwd'>]><Tagging/>"
+            .getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(signedUri(tagging, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var cleared = client.send(signedUri(tagging, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        if (!cleared.body().contains("<TagSet></TagSet>")) throw new AssertionError("Tag deletion failed");
+        status(200, client.send(signedUri(URI.create(base + "/objects/attributes-copy.txt"), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", "/objects/attributes.txt")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var copied = client.send(signed(base, "GET", "attributes-copy.txt", new byte[0]),
+            HttpResponse.BodyHandlers.ofByteArray());
+        if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse("")))
+            throw new AssertionError("Copy lost user metadata");
+        status(400, client.send(signedUri(object, "PUT", body,
+            Map.of("x-amz-meta-bad", "a".repeat(2100))), HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testBuckets(HttpClient client, String base) throws Exception {
+        var root = URI.create(base + "/");
+        var existing = client.send(signedUri(root, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        status(200, existing);
+        if (!existing.body().contains("<Name>objects</Name>"))
+            throw new AssertionError("Default bucket absent from service listing");
+        var bucket = URI.create(base + "/second-bucket");
+        status(200, client.send(signedUri(URI.create(bucket + "?x-id=CreateBucket"),
+            "PUT", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(409, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(200, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var upload = client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploads"),
+            "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+        status(200, upload);
+        String uploadId = upload.body().split("<UploadId>")[1].split("</UploadId>")[0];
+        status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploadId=" + uploadId),
+            "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+        byte[] body = "second bucket".getBytes(StandardCharsets.UTF_8);
+        var object = URI.create(base + "/second-bucket/one.txt");
+        status(200, client.send(signedUri(object, "PUT", body, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var read = client.send(signedUri(object, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, read);
+        if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Second bucket read failed");
+        var crossCopy = URI.create(base + "/second-bucket/copied.txt");
+        status(200, client.send(signedUri(crossCopy, "PUT", new byte[0],
+            Map.of("x-amz-copy-source", "/objects/attributes.txt")),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var copied = client.send(signedUri(crossCopy, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse("")))
+            throw new AssertionError("Cross-bucket copy lost metadata");
+        var listed = client.send(signedUri(URI.create(base + "/second-bucket?list-type=2"),
+            "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+        if (!listed.body().contains("<Key>one.txt</Key>")) throw new AssertionError("Second bucket listing failed");
+        status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(signedUri(crossCopy, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(204, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        status(404, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+    }
+
+    private static void testVersioning(HttpClient client, String base) throws Exception {
+        URI bucket = URI.create(base + "/version-bucket");
+        URI configuration = URI.create(bucket + "?versioning");
+        URI object = URI.create(bucket + "/note.txt");
+        status(200, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var initial = client.send(signedUri(configuration, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofString());
+        if (!initial.body().contains("<VersioningConfiguration") || initial.body().contains("<Status>"))
+            throw new AssertionError("New bucket versioning state");
+        status(200, client.send(signedUri(object, "PUT", new byte[]{1}, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        byte[] enable = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"
+            .getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var first = client.send(signedUri(object, "PUT", new byte[]{2}, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, first);
+        String id = first.headers().firstValue("x-amz-version-id").orElseThrow();
+        var second = client.send(signedUri(object, "PUT", new byte[]{3}, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, second);
+        if (id.equals(second.headers().firstValue("x-amz-version-id").orElseThrow()))
+            throw new AssertionError("Version IDs repeated");
+        var old = client.send(signedUri(URI.create(object + "?versionId=" + id), "GET",
+            new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+        status(200, old);
+        if (old.body()[0] != 2) throw new AssertionError("Historical object body");
+        byte[] tagged = "<Tagging><TagSet><Tag><Key>kind</Key><Value>old</Value></Tag></TagSet></Tagging>"
+            .getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "PUT",
+            tagged, Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+        var oldTags = client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "GET",
+            new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+        status(200, oldTags);
+        if (!oldTags.body().contains("<Value>old</Value>"))
+            throw new AssertionError("Versioned tagging failed");
+        var copied = client.send(signedUri(URI.create(bucket + "/copied.txt"), "PUT",
+            new byte[0], Map.of("x-amz-copy-source", "/version-bucket/note.txt?versionId=" + id)),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, copied);
+        if (!id.equals(copied.headers().firstValue("x-amz-copy-source-version-id").orElse("")))
+            throw new AssertionError("Copy did not report source version");
+        var copyBody = client.send(signedUri(URI.create(bucket + "/copied.txt"), "GET",
+            new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+        status(200, copyBody);
+        if (copyBody.body()[0] != 2) throw new AssertionError("Copy of old version failed");
+        var deleted = client.send(signedUri(object, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(204, deleted);
+        String marker = deleted.headers().firstValue("x-amz-version-id").orElseThrow();
+        var hidden = client.send(signedUri(object, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(404, hidden);
+        if (!"true".equals(hidden.headers().firstValue("x-amz-delete-marker").orElse("")))
+            throw new AssertionError("Missing delete marker response header");
+        status(405, client.send(signedUri(URI.create(object + "?versionId=" + marker), "GET",
+            new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+        var listed = client.send(signedUri(URI.create(bucket + "?versions&max-keys=2"), "GET",
+            new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+        status(200, listed);
+        if (!listed.body().contains("<DeleteMarker>") || !listed.body().contains("<IsTruncated>true</IsTruncated>"))
+            throw new AssertionError("Version listing did not include delete marker");
+        status(204, client.send(signedUri(URI.create(object + "?versionId=" + marker), "DELETE",
+            new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
+        var restored = client.send(signedUri(object, "GET", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, restored);
+        if (restored.body()[0] != 3) throw new AssertionError("Deleting marker did not restore current version");
+        byte[] suspend = "<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>"
+            .getBytes(StandardCharsets.UTF_8);
+        status(200, client.send(signedUri(configuration, "PUT", suspend, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var nullVersion = client.send(signedUri(object, "PUT", new byte[]{4}, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, nullVersion);
+        if (!"null".equals(nullVersion.headers().firstValue("x-amz-version-id").orElse("")))
+            throw new AssertionError("Suspended write did not produce null version");
+        status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        URI multipartObject = URI.create(bucket + "/multipart.txt");
+        var initiated = client.send(signedUri(URI.create(multipartObject + "?uploads"), "POST",
+            new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
+        status(200, initiated);
+        String uploadId = initiated.body().split("<UploadId>")[1].split("</UploadId>")[0];
+        byte[] partBody = "versioned multipart".getBytes(StandardCharsets.UTF_8);
+        var part = client.send(signedUri(URI.create(multipartObject + "?partNumber=1&uploadId=" + uploadId),
+            "PUT", partBody, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+        status(200, part);
+        String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
+            part.headers().firstValue("etag").orElseThrow() + "</ETag></Part></CompleteMultipartUpload>";
+        var completed = client.send(signedUri(URI.create(multipartObject + "?uploadId=" + uploadId),
+            "POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray());
+        status(200, completed);
+        String multipartVersion = completed.headers().firstValue("x-amz-version-id").orElseThrow();
+        status(204, client.send(signedUri(multipartObject, "DELETE", new byte[0], Map.of()),
+            HttpResponse.BodyHandlers.ofByteArray()));
+        var retainedMultipart = client.send(signedUri(URI.create(multipartObject + "?versionId=" +
+            multipartVersion), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray());
+        status(200, retainedMultipart);
+        if (!java.util.Arrays.equals(partBody, retainedMultipart.body()))
+            throw new AssertionError("Completed multipart version was not retained");
+    }
+
+    private static int virtualHostPut(int port, String host, String signedHost, byte[] body) throws Exception {
+        URI signed = URI.create("http://" + signedHost + ":" + port + "/photos/cat.jpg");
+        HttpRequest request = signedUri(signed, "PUT", body, Map.of());
+        String headers = "PUT /photos/cat.jpg HTTP/1.1\r\nHost: " + host + ":" + port +
+            "\r\nAuthorization: " + request.headers().firstValue("authorization").orElseThrow() +
+            "\r\nx-amz-date: " + request.headers().firstValue("x-amz-date").orElseThrow() +
+            "\r\nx-amz-content-sha256: " + request.headers().firstValue("x-amz-content-sha256").orElseThrow() +
+            "\r\nContent-Length: " + body.length + "\r\nConnection: close\r\n\r\n";
+        try (var socket = new java.net.Socket("127.0.0.1", port)) {
+            socket.setSoTimeout(5000);
+            socket.getOutputStream().write(headers.getBytes(StandardCharsets.US_ASCII));
+            socket.getOutputStream().write(body);
+            String response = new String(socket.getInputStream().readAllBytes(), StandardCharsets.ISO_8859_1);
+            return Integer.parseInt(response.split(" ", 3)[1]);
+        }
+    }
+
+    private static void testGatewayConcurrency(DiskStore store, HttpClient client,
+                                               java.util.concurrent.Executor executor) throws Exception {
+        HttpServer limited = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16);
+        var app = new Main(store, new MultipartStore(store),
+            new SigV4(ACCESS, SECRET, REGION, Clock.systemUTC()), "objects",
+            ClientLimits.disabled(), 1, "");
+        limited.setExecutor(executor);
+        limited.createContext("/", app::handle);
+        limited.start();
+        int port = limited.getAddress().getPort();
+        String base = "http://127.0.0.1:" + port;
+        byte[] body = {42};
+        HttpRequest request = signedUri(URI.create(base + "/objects/held"), "PUT", body, Map.of());
+        try (var socket = new java.net.Socket("127.0.0.1", port)) {
+            socket.setSoTimeout(5000);
+            String headers = "PUT /objects/held HTTP/1.1\r\nHost: 127.0.0.1:" + port +
+                "\r\nAuthorization: " + request.headers().firstValue("authorization").orElseThrow() +
+                "\r\nx-amz-date: " + request.headers().firstValue("x-amz-date").orElseThrow() +
+                "\r\nx-amz-content-sha256: " + request.headers().firstValue("x-amz-content-sha256").orElseThrow() +
+                "\r\nContent-Length: 1\r\nConnection: close\r\n\r\n";
+            socket.getOutputStream().write(headers.getBytes(StandardCharsets.US_ASCII));
+            boolean refused = false;
+            for (int attempt = 0; attempt < 50 && !refused; attempt++) {
+                int status = client.send(HttpRequest.newBuilder(URI.create(base + "/health")).GET().build(),
+                    HttpResponse.BodyHandlers.discarding()).statusCode();
+                refused = status == 503;
+                if (!refused) Thread.sleep(10);
+            }
+            if (!refused) throw new AssertionError("Configured gateway concurrency cap was not enforced");
+            socket.getOutputStream().write(body);
+            String finished = new String(socket.getInputStream().readAllBytes(), StandardCharsets.ISO_8859_1);
+            if (!finished.startsWith("HTTP/1.1 200 "))
+                throw new AssertionError("Held request did not complete after releasing its body");
+        } finally {
+            limited.stop(0);
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Path root = Files.createTempDirectory("store-http-test-");
+        var executor = Executors.newVirtualThreadPerTaskExecutor();
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16);
+        DiskStore store = new DiskStore(root, 1024, 4096);
+        try {
+            var app = new Main(store, new MultipartStore(store),
+                new SigV4(Map.of(ACCESS, SECRET, SECONDARY, SECONDARY_SECRET),
+                    ACCESS, REGION, Clock.systemUTC()), "objects", ClientLimits.disabled(), 16, "s3.test");
+            server.setExecutor(executor);
+            server.createContext("/", app::handle);
+            server.start();
+            String base = "http://127.0.0.1:" + server.getAddress().getPort();
+            HttpClient client = HttpClient.newHttpClient();
+            testCapabilities(client, base);
+            testPresigned(client, base);
+            testStreaming(client, base);
+            testStreamingTrailer(client, base);
+            testObjects(client, base);
+            testListing(client, base);
+            testCopy(client, base);
+            testChecksums(client, base);
+            testMultipart(client, base);
+            testAttributes(client, base);
+            testDelete(client, base);
+            testBuckets(client, base);
+            testVersioning(client, base);
+            testAcl(client, base);
+            byte[] hostedBody = "virtual host".getBytes(StandardCharsets.UTF_8);
+            int port = server.getAddress().getPort();
+            if (virtualHostPut(port, "objects.s3.test", "objects.s3.test", hostedBody) != 200)
+                throw new AssertionError("Signed virtual-hosted upload failed");
+            try (var opened = store.open("objects", "photos/cat.jpg")) {
+                if (!java.util.Arrays.equals(hostedBody, opened.stream().readAllBytes()))
+                    throw new AssertionError("Virtual-hosted bucket or key was parsed incorrectly");
+            }
+            if (virtualHostPut(port, "objects.s3.test", "other.s3.test", hostedBody) != 403)
+                throw new AssertionError("Changing a signed virtual hostname was accepted");
+            testGatewayConcurrency(store, client, executor);
+            System.out.println("HTTP tests passed: capabilities, objects, copy, checksums, listing, multipart, attributes, buckets, versioning, ACLs");
         } finally {
             server.stop(0);
             executor.close();

test/cloud/lunarsky/store/StoreTest.java

@@ -5,32 +5,120 @@ import java.util.Arrays;
 import java.nio.ByteBuffer;
 import java.security.MessageDigest;
 import java.util.List;
+import java.util.Map;
 
 public final class StoreTest {
     interface Operation {void run() throws Exception;}
     static void fails(int status,Operation operation)throws Exception{
-        try{operation.run();throw new AssertionError("Expected "+status);}catch(StoreException error){if(error.status!=status)throw error;}
+        try {
+            operation.run();
+            throw new AssertionError("Expected " + status);
+        } catch (StoreException error) {
+            if (error.status != status) throw error;
+        }
     }
     static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
         return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream");
     }
-    public static void main(String[] args)throws Exception{
+    private static void testSignature() throws Exception {
         var headers=new com.sun.net.httpserver.Headers();
-        headers.set("host","examplebucket.s3.amazonaws.com");headers.set("range","bytes=0-9");
+        headers.set("host","examplebucket.s3.amazonaws.com");
+        headers.set("range","bytes=0-9");
         headers.set("x-amz-date","20130524T000000Z");
         headers.set("x-amz-content-sha256","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
         headers.set("authorization","AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41");
         var clock=java.time.Clock.fixed(java.time.Instant.parse("2013-05-24T00:00:00Z"),java.time.ZoneOffset.UTC);
         var auth=new SigV4("AKIAIOSFODNN7EXAMPLE","wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","us-east-1",clock);
-        var uri=java.net.URI.create("/test.txt");auth.verify("GET",uri,headers);
+        var uri=java.net.URI.create("/test.txt");
+        auth.verify("GET",uri,headers);
         fails(403,()->auth.verify("GET",java.net.URI.create("/other.txt"),headers));
         fails(403,()->auth.verify("DELETE",uri,headers));
         fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",clock).verify("GET",uri,headers));
         fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers));
-        headers.add("host","duplicate");fails(403,()->auth.verify("GET",uri,headers));
+        headers.add("host","duplicate");
+        fails(403,()->auth.verify("GET",uri,headers));
+        var presignedHeaders = new com.sun.net.httpserver.Headers();
+        presignedHeaders.set("host", "examplebucket.s3.amazonaws.com");
+        var presigned = java.net.URI.create("/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256" +
+            "&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request" +
+            "&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host" +
+            "&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404");
+        if (!"UNSIGNED-PAYLOAD".equals(auth.verifyRequest("GET", presigned, presignedHeaders).payload()))
+            throw new AssertionError("Official presigned URL was not accepted");
+        fails(403, () -> auth.verifyRequest("PUT", presigned, presignedHeaders));
+        fails(403, () -> new SigV4("AKIAIOSFODNN7EXAMPLE",
+            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1",
+            java.time.Clock.fixed(java.time.Instant.parse("2013-05-25T00:00:01Z"),
+                java.time.ZoneOffset.UTC)).verifyRequest("GET", presigned, presignedHeaders));
         System.out.println("SigV4 official vector and tampering tests passed");
-        Path root=Files.createTempDirectory("store-test-");
+    }
-        try{
+
+    private static void testAwsChunked() throws Exception {
+        String secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
+        String date = "20130524T000000Z";
+        String scope = "20130524/us-east-1/s3/aws4_request";
+        byte[] key = SigV4.signingKey(secret, "20130524", "us-east-1");
+        byte[] body = new byte[66560];
+        Arrays.fill(body, (byte) 'a');
+        var encoded = new ByteArrayOutputStream();
+        encoded.write("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n".getBytes());
+        encoded.write(body, 0, 65536);
+        encoded.write("\r\n400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n".getBytes());
+        encoded.write(body, 65536, 1024);
+        encoded.write("\r\n0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n".getBytes());
+        var authorization = new SigV4.Verified("STREAMING-AWS4-HMAC-SHA256-PAYLOAD", "", key,
+            date, scope, "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9");
+        try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(encoded.toByteArray()),
+                authorization, body.length, null)) {
+            if (!Arrays.equals(body, stream.readAllBytes())) throw new AssertionError("Signed chunk vector mismatch");
+        }
+        byte[] tampered = encoded.toByteArray();
+        tampered[100] = 'b';
+        fails(400, () -> {
+            try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(tampered),
+                    authorization, body.length, null)) { stream.readAllBytes(); }
+        });
+        var withTrailer = new ByteArrayOutputStream();
+        withTrailer.write("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n".getBytes());
+        withTrailer.write(body, 0, 65536);
+        withTrailer.write("\r\n400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n".getBytes());
+        withTrailer.write(body, 65536, 1024);
+        withTrailer.write("\r\n0;chunk-signature=2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992\r\n".getBytes());
+        withTrailer.write("x-amz-checksum-crc32c:sOO8/Q==\r\n".getBytes());
+        withTrailer.write("x-amz-trailer-signature=d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435\r\n\r\n".getBytes());
+        var trailerAuthorization = new SigV4.Verified("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER",
+            "", key, date, scope, "106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e");
+        try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(withTrailer.toByteArray()),
+                trailerAuthorization, body.length, "x-amz-checksum-crc32c")) {
+            if (!Arrays.equals(body, stream.readAllBytes()) || !"sOO8/Q==".equals(stream.trailerValue()))
+                throw new AssertionError("Signed checksum trailer vector mismatch");
+        }
+    }
+
+    private static void testCrc64Nvme() {
+        byte[] zeros = new byte[32];
+        Crc64Nvme checksum = new Crc64Nvme();
+        checksum.update(zeros, 0, zeros.length);
+        if (checksum.getValue() != 0xcf3473434d4ecf3bL)
+            throw new AssertionError("CRC64NVME test vector mismatch");
+    }
+
+    private static void testXxHashes() {
+        byte[] body = "abc".getBytes(java.nio.charset.StandardCharsets.US_ASCII);
+        Map<String, String> vectors = Map.of(
+            "XXHASH64", "44bc2cf5ad770999",
+            "XXHASH3", "78af5f94892f3950",
+            "XXHASH128", "06b05ab6733a618578af5f94892f3950");
+        for (var entry : vectors.entrySet()) {
+            XxHashes hash = new XxHashes(entry.getKey());
+            hash.update(body, 0, 1);
+            hash.update(body, 1, 2);
+            if (!SigV4.hex(hash.digest()).equals(entry.getValue()))
+                throw new AssertionError(entry.getKey() + " test vector mismatch");
+        }
+    }
+
+    private static void testInitialStore(Path root) throws Exception {
         try(var store=new DiskStore(root,8,10)){
             byte[] body={1,2,3,4,5,6};
             put(store,"../nested/☾",body,true);
@@ -52,10 +140,14 @@ public final class StoreTest {
                 if(!expected.getMessage().contains("already in use"))throw expected;
             }
         }
+    }
+
+    private static void testRestart(Path root) throws Exception {
         try(var restarted=new DiskStore(root,8,10)){
             try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
             if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
-                restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
+            restarted.delete("test","../nested/☾");
+            restarted.delete("test","../nested/☾");
             fails(404,()->restarted.open("test","../nested/☾"));
             var uploads=new MultipartStore(restarted);
             String upload=uploads.create("test","from-parts","text/plain");
@@ -64,6 +156,9 @@ public final class StoreTest {
                 SigV4.hex(SigV4.hash(part)),null);
             Files.writeString(root.resolve("pending-upload-id"),upload);
         }
+    }
+
+    private static void testMultipartRecovery(Path root) throws Exception {
         try(var resumed=new DiskStore(root,8,10)){
             Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000");
             Files.createDirectory(unfinished);
@@ -80,6 +175,9 @@ public final class StoreTest {
             }
             resumed.delete("test","from-parts");
         }
+    }
+
+    private static void testLegacyRecord(Path root) throws Exception {
         byte[] old={4,5,6};
         String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
         Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId);
@@ -100,6 +198,9 @@ public final class StoreTest {
             if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy")))
                 throw new AssertionError("Legacy overwrite was not indexed");
         }
+    }
+
+    private static void testCorruption(Path root) throws Exception {
         Files.delete(root.resolve("pending-upload-id"));
         String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
         Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
@@ -110,7 +211,227 @@ public final class StoreTest {
             if(!expected.getMessage().contains("object record"))throw expected;
         }
         try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
+    }
+
+    private static void testAttributesRestart(Path root) throws Exception {
+        Path data = root.resolve("attributes");
+        byte[] body = {1, 2, 3};
+        var crc = new Crc64Nvme();
+        crc.update(body, 0, body.length);
+        byte[] digest = ByteBuffer.allocate(8).putLong(crc.getValue()).array();
+        Map<String, String> checksums = Map.of("x-amz-checksum-crc64nvme",
+            java.util.Base64.getEncoder().encodeToString(digest));
+        try (var store = new DiskStore(data, 8, 10)) {
+            store.put("test", "metadata", new ByteArrayInputStream(body), body.length,
+                SigV4.hex(SigV4.hash(body)), null, false, "text/plain",
+                Map.of("project", "LunarSky"), Map.of("stage", "one"), () -> checksums);
+            store.setTags("test", "metadata", Map.of("stage", "two"));
+        }
+        try (var store = new DiskStore(data, 8, 10);
+             var object = store.open("test", "metadata")) {
+            if (!Arrays.equals(body, object.stream().readAllBytes()) ||
+                !object.metadata().userMetadata().equals(Map.of("project", "LunarSky")) ||
+                !object.metadata().tags().equals(Map.of("stage", "two")) ||
+                !object.metadata().checksums().equals(checksums))
+                throw new AssertionError("Object attributes were lost after restart");
+        }
+    }
+
+    private static void testV3Record(Path root) throws Exception {
+        Path data = root.resolve("v3-record");
+        String bucket = "test", key = "v3-object";
+        byte[] body = {8, 9, 10};
+        byte[] bucketBytes = bucket.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        byte[] keyBytes = key.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        byte[] typeBytes = "text/plain".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        byte[] metadata = ObjectAttributes.encode(Map.of("legacy", "yes"), 4096);
+        byte[] tags = ObjectAttributes.encode(Map.of("source", "v3"), 8192);
+        String pathHash = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(java.nio.charset.StandardCharsets.UTF_8)));
+        Path path = data.resolve("objects").resolve(pathHash.substring(0, 2)).resolve(pathHash);
+        Files.createDirectories(path.getParent());
+        ByteBuffer bytes = ByteBuffer.allocate(82 + bucketBytes.length + keyBytes.length +
+            typeBytes.length + metadata.length + tags.length + body.length);
+        bytes.putLong(0x4c534f424a303033L).putLong(body.length).putLong(123456789L)
+            .put(MessageDigest.getInstance("MD5").digest(body)).put(SigV4.hash(body))
+            .putShort((short) bucketBytes.length).putShort((short) keyBytes.length)
+            .putShort((short) typeBytes.length).putShort((short) metadata.length)
+            .putShort((short) tags.length).put(bucketBytes).put(keyBytes).put(typeBytes)
+            .put(metadata).put(tags).put(body);
+        Files.write(path, bytes.array());
+        try (var store = new DiskStore(data, 64, 128)) {
+            try (var object = store.open(bucket, key)) {
+                if (!Arrays.equals(body, object.stream().readAllBytes()) ||
+                    !object.metadata().userMetadata().equals(Map.of("legacy", "yes")) ||
+                    !object.metadata().tags().equals(Map.of("source", "v3")))
+                    throw new AssertionError("V3 record was not readable");
+            }
+            store.setTags(bucket, key, Map.of("source", "v4"));
+        }
+        try (var store = new DiskStore(data, 64, 128);
+             var object = store.open(bucket, key)) {
+            if (!Arrays.equals(body, object.stream().readAllBytes()) ||
+                !object.metadata().tags().equals(Map.of("source", "v4")))
+                throw new AssertionError("V3 record upgrade failed");
+        }
+    }
+
+    private static void testBucketRestart(Path root) throws Exception {
+        Path data = root.resolve("buckets");
+        byte[] body = {4, 5, 6};
+        try (var store = new DiskStore(data, 8, 10)) {
+            store.ensureBucket("default-bucket");
+            store.createBucket("second-bucket");
+            store.put("second-bucket", "object", new ByteArrayInputStream(body), body.length,
+                SigV4.hex(SigV4.hash(body)), null, false, "application/octet-stream");
+        }
+        try (var store = new DiskStore(data, 8, 10);
+             var object = store.open("second-bucket", "object")) {
+            if (store.buckets().size() != 2 || !Arrays.equals(body, object.stream().readAllBytes()))
+                throw new AssertionError("Bucket catalog was lost after restart");
+            fails(409, () -> store.deleteBucket("second-bucket"));
+            store.delete("second-bucket", "object");
+            store.deleteBucket("second-bucket");
+            fails(404, () -> store.bucket("second-bucket"));
+        }
+    }
+
+    private static void testAclPersistence(Path root) throws Exception {
+        Path data = root.resolve("acl");
+        Map<String, String> grant = Map.of("SECONDARYKEY1234", Integer.toString(Acl.READ));
+        byte[] body = {9, 8, 7};
+        String older;
+        try (var store = new DiskStore(data, 32, 128)) {
+            store.ensureBucket("acl-bucket");
+            store.setBucketAcl("acl-bucket", grant);
+            store.setVersioning("acl-bucket", ObjectStorage.VersioningState.ENABLED);
+            older = store.put("acl-bucket", "image", new ByteArrayInputStream(body), body.length,
+                SigV4.hex(SigV4.hash(body)), null, false, "image/png",
+                Map.of(), Map.of(), Map::of, grant).versionId();
+            store.put("acl-bucket", "image", new ByteArrayInputStream(body), body.length,
+                SigV4.hex(SigV4.hash(body)), null, false, "image/png");
+        }
+        try (var store = new DiskStore(data, 32, 128);
+             var previous = store.open("acl-bucket", "image", older);
+             var current = store.open("acl-bucket", "image")) {
+            if (!store.bucket("acl-bucket").acl().equals(grant) ||
+                !previous.metadata().acl().equals(grant) || !current.metadata().acl().isEmpty())
+                throw new AssertionError("ACL grants changed after restart or version replacement");
+            store.setObjectAcl("acl-bucket", "image", older, Map.of(Acl.ALL_USERS, "1"));
+        }
+        try (var store = new DiskStore(data, 32, 128);
+             var previous = store.open("acl-bucket", "image", older)) {
+            if (!previous.metadata().acl().equals(Map.of(Acl.ALL_USERS, "1")))
+                throw new AssertionError("Version-specific ACL edit was not durable");
+        }
+    }
+
+    private static void testAdditionalKeys(Path root) throws Exception {
+        Path file = root.resolve("access-keys");
+        Files.writeString(file, "SECONDARYKEY1234:secondary-secret-key-that-is-at-least-32-characters\n");
+        Map<String, String> keys = Main.identities(
+            Map.of("S3_CREDENTIALS_FILE", file.toString()),
+            "TESTACCESSKEY123", "test-secret-key-that-is-at-least-32-characters");
+        if (keys.size() != 2 || !keys.containsKey("SECONDARYKEY1234"))
+            throw new AssertionError("Additional access key was not loaded");
+        Files.writeString(file, "TESTACCESSKEY123:duplicate-root-secret-that-is-at-least-32-characters\n");
+        try {
+            Main.identities(Map.of("S3_CREDENTIALS_FILE", file.toString()),
+                "TESTACCESSKEY123", "test-secret-key-that-is-at-least-32-characters");
+            throw new AssertionError("Duplicate root key was accepted");
+        } catch (IllegalArgumentException expected) { }
+    }
+
+    private static void testVersioning(Path root) throws Exception {
+        Path data = root.resolve("versioning");
+        String first;
+        String second;
+        String marker;
+        try (var store = new DiskStore(data, 8, 100)) {
+            store.createBucket("versioned-bucket");
+            byte[] old = {1};
+            store.put("versioned-bucket", "note", new ByteArrayInputStream(old), old.length,
+                SigV4.hex(SigV4.hash(old)), null, false, "text/plain");
+            store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.ENABLED);
+            byte[] newer = {2};
+            first = store.put("versioned-bucket", "note", new ByteArrayInputStream(newer), newer.length,
+                SigV4.hex(SigV4.hash(newer)), null, false, "text/plain").versionId();
+            byte[] latest = {3};
+            second = store.put("versioned-bucket", "note", new ByteArrayInputStream(latest), latest.length,
+                SigV4.hex(SigV4.hash(latest)), null, false, "text/plain").versionId();
+            if (first == null || second == null || first.equals(second)) throw new AssertionError("Unique versions");
+            if (store.usedBytes() != 3) throw new AssertionError("Retained versions did not count toward capacity");
+            try (var object = store.open("versioned-bucket", "note", first)) {
+                if (object.stream().read() != 2) throw new AssertionError("Old version was overwritten");
+            }
+            marker = store.delete("versioned-bucket", "note", null).versionId();
+            fails(404, () -> store.open("versioned-bucket", "note"));
+            if (!store.list("versioned-bucket", "", "", 10, null).objects().isEmpty())
+                throw new AssertionError("Delete marker appeared in current listing");
+            var page = store.listVersions("versioned-bucket", "", null, null, 2);
+            if (!page.truncated() || !page.entries().getFirst().deleteMarker() ||
+                !page.entries().get(1).versionId().equals(second)) throw new AssertionError("Version listing");
+            var rest = store.listVersions("versioned-bucket", "", page.nextKey(), page.nextVersionId(), 10);
+            if (rest.entries().size() != 2 || !rest.entries().getLast().versionId().equals("null"))
+                throw new AssertionError("Version pagination");
+            store.delete("versioned-bucket", "note", marker);
+            try (var object = store.open("versioned-bucket", "note")) {
+                if (object.stream().read() != 3) throw new AssertionError("Delete marker removal");
+            }
+            store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.SUSPENDED);
+            byte[] suspended = {4};
+            var nullVersion = store.put("versioned-bucket", "note", new ByteArrayInputStream(suspended),
+                suspended.length, SigV4.hex(SigV4.hash(suspended)), null, false, "text/plain");
+            if (!"null".equals(nullVersion.versionId())) throw new AssertionError("Suspended null version");
+            if (store.usedBytes() != 3) throw new AssertionError("Suspended write did not replace null version");
+            try (var object = store.open("versioned-bucket", "note", second)) {
+                if (object.stream().read() != 3) throw new AssertionError("Suspension removed a version");
+            }
+            store.setTags("versioned-bucket", "note", Map.of("stage", "suspended"));
+        }
+        try (var store = new DiskStore(data, 8, 100)) {
+            if (store.bucket("versioned-bucket").versioning() != ObjectStorage.VersioningState.SUSPENDED)
+                throw new AssertionError("Versioning state was lost");
+            try (var object = store.open("versioned-bucket", "note")) {
+                if (object.stream().read() != 4 || !object.metadata().tags().equals(Map.of("stage", "suspended")))
+                    throw new AssertionError("Versioned object was lost");
+            }
+            store.delete("versioned-bucket", "note");
+            fails(404, () -> store.open("versioned-bucket", "note"));
+            if (store.usedBytes() != 2) throw new AssertionError("Suspended delete did not release null version");
+            try (var object = store.open("versioned-bucket", "note", second)) {
+                if (object.stream().read() != 3) throw new AssertionError("Suspended delete removed old version");
+            }
+            fails(409, () -> store.deleteBucket("versioned-bucket"));
+            for (var entry : store.listVersions("versioned-bucket", "", null, null, 10).entries())
+                store.delete("versioned-bucket", "note", entry.versionId());
+            if (store.usedBytes() != 0) throw new AssertionError("Version deletes did not release capacity");
+            store.deleteBucket("versioned-bucket");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        testSignature();
+        testAwsChunked();
+        testCrc64Nvme();
+        testXxHashes();
+        Path root = Files.createTempDirectory("store-test-");
+        try {
+            testInitialStore(root);
+            testRestart(root);
+            testMultipartRecovery(root);
+            testLegacyRecord(root);
+            testAttributesRestart(root);
+            testV3Record(root);
+            testBucketRestart(root);
+            testAclPersistence(root);
+            testAdditionalKeys(root);
+            testVersioning(root);
+            testCorruption(root);
             System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete");
-        }finally{try(var paths=Files.walk(root)){for(var p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}}
+        } finally {
+            try (var paths = Files.walk(root)) {
+                for (var path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) Files.delete(path);
+            }
+        }
     }
 }

tests/metadata/compose.yaml

@@ -0,0 +1,30 @@
+services:
+  metadata:
+    image: postgres:17-alpine
+    environment:
+      POSTGRES_DB: objectstore_test
+      POSTGRES_USER: objectstore_test
+      POSTGRES_PASSWORD: local-metadata-test-only
+    tmpfs:
+      - /var/lib/postgresql/data:size=268435456
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U objectstore_test -d objectstore_test"]
+      interval: 2s
+      timeout: 2s
+      retries: 20
+    mem_limit: 256m
+
+  check:
+    build:
+      context: ../..
+    image: lunarsky-objectstore:metadata-test
+    entrypoint: ["java", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar:/app/hash4j.jar", "cloud.lunarsky.store.ClusterMetadataTest"]
+    environment:
+      POSTGRES_JDBC_URL: jdbc:postgresql://127.0.0.2:5432,metadata:5432/objectstore_test?connectTimeout=1&socketTimeout=10&targetServerType=primary&hostRecheckSeconds=0
+      POSTGRES_ADMIN_JDBC_URL: jdbc:postgresql://metadata:5432/postgres?connectTimeout=3&socketTimeout=10
+      POSTGRES_USER: objectstore_test
+      POSTGRES_PASSWORD: local-metadata-test-only
+    depends_on:
+      metadata:
+        condition: service_healthy
+    mem_limit: 384m

tests/two-host/README.md

@@ -0,0 +1,50 @@
+# Two-machine durability drill
+
+Run this disposable test on two machines. Machine A runs the gateway, PostgreSQL, and one storage node; machine B runs a second storage node. Keep the node connection on a private network: this drill uses bearer tokens over HTTP and does not enable the optional node TLS setup.
+
+Both machines need Docker. Machine A also needs Docker Compose and Python 3. The example uses loopback port 9003 on machine A and private-network port 9103 on machine B; change them if needed. Use separate test volumes, and do not point this drill at an existing ObjectStore cluster.
+
+## Start the cluster
+
+On machine A, generate test-only credentials outside the repository and build the current image:
+
+```sh
+python3 tests/two-host/prepare.py /path/to/private-test-dir http://MACHINE_B_PRIVATE_IP:9103
+docker build -t objectstore-durability:local .
+docker save objectstore-durability:local | gzip > /path/to/private-test-dir/objectstore-durability.tar.gz
+```
+
+Transfer the image archive and `remote-node.env` to a private directory on machine B. Load the image there, then start its node with a dedicated volume and a port bound only to its private-network address:
+
+```sh
+gzip -dc objectstore-durability.tar.gz | sudo docker load
+sudo docker run -d --name objectstore-durability-remote --restart unless-stopped \
+  --publish MACHINE_B_PRIVATE_IP:9103:9100 \
+  --volume objectstore-durability-remote-data:/data \
+  --env-file remote-node.env --entrypoint java objectstore-durability:local \
+  --add-modules jdk.httpserver,java.net.http -cp /app:/app/postgresql.jar \
+  cloud.lunarsky.store.ClusterNode
+```
+
+Start the services on machine A from the repository root:
+
+```sh
+docker compose --env-file /path/to/private-test-dir/cluster.env \
+  -f tests/two-host/compose.yaml up -d --wait gateway
+python3 tests/two-host/check.py /path/to/private-test-dir/cluster.env \
+  /path/to/private-test-dir/acknowledged.jsonl seed
+python3 tests/two-host/check.py /path/to/private-test-dir/cluster.env \
+  /path/to/private-test-dir/acknowledged.jsonl verify
+```
+
+Before disrupting either machine, check `cluster_segments.replica_ids` against `cluster_nodes.host_id` in the test PostgreSQL database. Every seeded segment must have replicas on two distinct host IDs. The IDs are operator labels; confirm that the nodes run on separate machines.
+
+## Failure checks
+
+Run `check.py ... stress --seconds 120` while both nodes are healthy, then interrupt machine B. The journal records a write only after an HTTP 200 response and calls `fsync` for each entry. New writes should return 503 while only one machine remains. Reads of acknowledged objects should still succeed from the other replica. Restore machine B, wait for its node to answer `/health`, and run `check.py ... verify` against the entire journal.
+
+Stop the node container on machine A and repeat the rejection and read checks using the replica on machine B. Restart the node, then abruptly kill only the disposable gateway, metadata, and node containers on machine A during another stress run. Start them again and verify the journal. A connection closed during a write has an **uncertain** outcome; do not count it as acknowledged or assume it was rejected.
+
+For a manual metadata recovery drill, take a custom-format `pg_dump` after the last acknowledged write and verify it with `pg_restore --list`. Copy the dump to machine B and restore it into a fresh PostgreSQL volume there. Stop the original gateway and metadata container on machine A. Start a separate gateway against the restored database, then verify the journal. Finally stop the storage node on machine A and verify again: this forces reads to use both the restored metadata and the replica on machine B. Keep the backup and its credentials private.
+
+Passing this drill does not prove every crash point, disk-controller write behavior, long-term bit-rot resistance, automatic metadata failover, or production readiness. The cluster still requires a manual metadata restore.

tests/two-host/check.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+import argparse
+import datetime
+import hashlib
+import hmac
+import http.client
+import json
+import os
+from pathlib import Path
+import time
+import urllib.parse
+
+
+def load_env(path):
+    return dict(line.split("=", 1) for line in Path(path).read_text().splitlines()
+                if line and not line.startswith("#"))
+
+
+def sign(key, value):
+    return hmac.new(key, value.encode(), hashlib.sha256).digest()
+
+
+def request(env, method, key, body=b""):
+    port = int(env.get("CLUSTER_HOST_PORT", "9003"))
+    host = f"127.0.0.1:{port}"
+    path = "/durability/" + urllib.parse.quote(key, safe="/-_.~")
+    date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    stamp = date[:8]
+    digest = hashlib.sha256(body).hexdigest()
+    headers = {"host": host, "x-amz-content-sha256": digest, "x-amz-date": date}
+    signed = ";".join(sorted(headers))
+    canonical_headers = "".join(f"{name}:{headers[name]}\n" for name in sorted(headers))
+    canonical = f"{method}\n{path}\n\n{canonical_headers}\n{signed}\n{digest}"
+    scope = f"{stamp}/us-east-1/s3/aws4_request"
+    to_sign = f"AWS4-HMAC-SHA256\n{date}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}"
+    key_bytes = ("AWS4" + env["S3_SECRET_KEY"]).encode()
+    for component in (stamp, "us-east-1", "s3", "aws4_request"):
+        key_bytes = sign(key_bytes, component)
+    signature = hmac.new(key_bytes, to_sign.encode(), hashlib.sha256).hexdigest()
+    headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={env['S3_ACCESS_KEY']}/{scope},"
+                                f"SignedHeaders={signed},Signature={signature}")
+    connection = http.client.HTTPConnection("127.0.0.1", port, timeout=40)
+    try:
+        connection.request(method, path, body=body if method == "PUT" else None, headers=headers)
+        response = connection.getresponse()
+        return response.status, response.read()
+    finally:
+        connection.close()
+
+
+def payload(key, length):
+    return hashlib.shake_256(key.encode()).digest(length)
+
+
+def record(journal, key, body):
+    entry = {"key": key, "length": len(body), "sha256": hashlib.sha256(body).hexdigest()}
+    with open(journal, "a", encoding="utf-8") as output:
+        output.write(json.dumps(entry, separators=(",", ":")) + "\n")
+        output.flush()
+        os.fsync(output.fileno())
+
+
+def put_and_record(env, journal, key, length):
+    body = payload(key, length)
+    try:
+        status, response = request(env, "PUT", key, body)
+    except (OSError, TimeoutError) as error:
+        return "uncertain", str(error)
+    if status == 200:
+        record(journal, key, body)
+        return "acknowledged", ""
+    return "rejected", f"HTTP {status}: {response[:120]!r}"
+
+
+def seed(env, journal):
+    for key, length in (("durability/seed-small", 4096),
+                        ("durability/seed-multisegment", 9 * 1024 * 1024 + 17)):
+        outcome, detail = put_and_record(env, journal, key, length)
+        if outcome != "acknowledged":
+            raise RuntimeError(f"Seed {key}: {outcome} {detail}")
+        print(f"Acknowledged {key}: {length} bytes", flush=True)
+
+
+def stress(env, journal, seconds):
+    end = time.monotonic() + seconds
+    counts = {"acknowledged": 0, "rejected": 0, "uncertain": 0}
+    sequence = 0
+    nonce = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S")
+    while time.monotonic() < end and sequence < 200:
+        key = f"durability/stress/{nonce}-{sequence:04d}"
+        outcome, detail = put_and_record(env, journal, key, 256 * 1024)
+        counts[outcome] += 1
+        if outcome != "acknowledged" and counts[outcome] == 1:
+            print(f"{key}: {outcome} {detail}", flush=True)
+        sequence += 1
+        time.sleep(0.1)
+    print(json.dumps(counts, sort_keys=True), flush=True)
+
+
+def verify(env, journal):
+    entries = [json.loads(line) for line in Path(journal).read_text().splitlines() if line]
+    if not entries:
+        raise RuntimeError("Journal contains no acknowledged writes")
+    failures = []
+    for entry in entries:
+        try:
+            status, body = request(env, "GET", entry["key"])
+            actual = hashlib.sha256(body).hexdigest()
+            if status != 200 or len(body) != entry["length"] or actual != entry["sha256"]:
+                failures.append(f"{entry['key']}: HTTP {status}, {len(body)} bytes, SHA-256 {actual}")
+        except (OSError, TimeoutError) as error:
+            failures.append(f"{entry['key']}: {error}")
+    for failure in failures:
+        print(failure)
+    print(f"Verified {len(entries) - len(failures)}/{len(entries)} acknowledged writes", flush=True)
+    if failures:
+        raise SystemExit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("env_file")
+    parser.add_argument("journal")
+    parser.add_argument("action", choices=("seed", "stress", "verify"))
+    parser.add_argument("--seconds", type=int, default=90)
+    args = parser.parse_args()
+    env = load_env(args.env_file)
+    if args.action == "seed":
+        seed(env, args.journal)
+    elif args.action == "stress":
+        stress(env, args.journal, args.seconds)
+    else:
+        verify(env, args.journal)
+
+
+if __name__ == "__main__":
+    main()

tests/two-host/compose.yaml

@@ -0,0 +1,120 @@
+services:
+  metadata:
+    image: postgres:17-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: objectstore
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    volumes:
+      - metadata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U objectstore -d objectstore"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    mem_limit: 256m
+    security_opt:
+      - no-new-privileges:true
+
+  node-local:
+    image: objectstore-durability:local
+    restart: unless-stopped
+    entrypoint: ["java", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.ClusterNode"]
+    environment:
+      CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
+      CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN}
+      CLUSTER_HOST_ID: ${LOCAL_HOST_ID:?Set LOCAL_HOST_ID}
+      NODE_BIND: 0.0.0.0
+      DATA_DIR: /data
+    volumes:
+      - node-local:/data
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    mem_limit: 256m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
+  gateway:
+    image: objectstore-durability:local
+    restart: unless-stopped
+    environment:
+      STORE_MODE: cluster
+      CLUSTER_LOCAL_DEV: "true"
+      BIND_ADDRESS: 0.0.0.0
+      S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
+      S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
+      S3_BUCKET: durability
+      S3_REGION: us-east-1
+      MAX_OBJECT_BYTES: 67108864
+      MAX_TOTAL_BYTES: 536870912
+      CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
+      CLUSTER_NODES: http://node-local:9100,${REMOTE_NODE_URL:?Set REMOTE_NODE_URL}
+      POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    ports:
+      - "127.0.0.1:${CLUSTER_HOST_PORT:-9003}:9000"
+    depends_on:
+      metadata:
+        condition: service_healthy
+      node-local:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9000/ready"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    mem_limit: 384m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
+  repair:
+    image: objectstore-durability:local
+    profiles: [repair]
+    entrypoint: ["/usr/local/bin/objectstore", "cluster-repair"]
+    environment:
+      STORE_MODE: cluster
+      CLUSTER_LOCAL_DEV: "true"
+      CLUSTER_NODES: http://node-local:9100,${REMOTE_NODE_URL:?Set REMOTE_NODE_URL}
+      CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN}
+      CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN}
+      S3_BUCKET: durability
+      POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    mem_limit: 384m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
+  metadata-recovery:
+    image: postgres:17-alpine
+    profiles: [recovery]
+    environment:
+      POSTGRES_DB: objectstore
+      POSTGRES_USER: objectstore
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
+    volumes:
+      - metadata-recovery:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U objectstore -d objectstore"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    mem_limit: 256m
+    security_opt:
+      - no-new-privileges:true
+
+volumes:
+  metadata:
+  metadata-recovery:
+  node-local:

tests/two-host/prepare.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+import argparse
+import os
+from pathlib import Path
+import secrets
+import urllib.parse
+import uuid
+
+
+def write_private(path, lines):
+    with path.open("x", encoding="utf-8") as output:
+        output.write("\n".join(lines) + "\n")
+    path.chmod(0o600)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory", type=Path)
+    parser.add_argument("remote_node_url")
+    parser.add_argument("--gateway-port", type=int, default=9003)
+    args = parser.parse_args()
+    url = urllib.parse.urlparse(args.remote_node_url)
+    if url.scheme != "http" or not url.hostname or not url.port or url.path or url.query or url.fragment:
+        parser.error("remote_node_url must be an http://host:port address without a path")
+    if not 1 <= args.gateway_port <= 65535:
+        parser.error("gateway port must be between 1 and 65535")
+    args.directory.mkdir(mode=0o700, parents=True, exist_ok=True)
+    args.directory.chmod(0o700)
+    access = "DurabilityTest" + secrets.token_hex(8)
+    secret = secrets.token_hex(32)
+    token = secrets.token_hex(32)
+    repair = secrets.token_hex(32)
+    remote_id = uuid.uuid4()
+    old_umask = os.umask(0o077)
+    try:
+        write_private(args.directory / "cluster.env", [
+            "COMPOSE_PROJECT_NAME=objectstore-durability",
+            f"S3_ACCESS_KEY={access}", f"S3_SECRET_KEY={secret}",
+            f"CLUSTER_TOKEN={token}", f"CLUSTER_REPAIR_TOKEN={repair}",
+            f"POSTGRES_PASSWORD={secrets.token_hex(24)}",
+            f"LOCAL_HOST_ID={uuid.uuid4()}", f"REMOTE_HOST_ID={remote_id}",
+            f"REMOTE_NODE_URL={args.remote_node_url}",
+            f"CLUSTER_HOST_PORT={args.gateway_port}", "S3_BUCKET=durability",
+        ])
+        write_private(args.directory / "remote-node.env", [
+            f"CLUSTER_TOKEN={token}", f"CLUSTER_REPAIR_TOKEN={repair}",
+            f"CLUSTER_HOST_ID={remote_id}", "NODE_BIND=0.0.0.0",
+            "NODE_PORT=9100", "DATA_DIR=/data",
+        ])
+    finally:
+        os.umask(old_umask)
+    print(f"Test configuration written to {args.directory}")
+
+
+if __name__ == "__main__":
+    main()