diff --git a/.env.cluster.example b/.env.cluster.example new file mode 100644 index 0000000..bf17b78 --- /dev/null +++ b/.env.cluster.example @@ -0,0 +1,7 @@ +S3_ACCESS_KEY= +S3_SECRET_KEY= +CLUSTER_TOKEN= +CLUSTER_REPAIR_TOKEN= +POSTGRES_PASSWORD= +S3_BUCKET=objects +CLUSTER_HOST_PORT=9001 diff --git a/.env.example b/.env.example index bc696b5..eec2e2b 100644 --- a/.env.example +++ b/.env.example @@ -5,5 +5,5 @@ S3_SECRET_KEY= S3_BUCKET=objects S3_REGION=us-east-1 HOST_PORT=9000 -MAX_OBJECT_BYTES=10485760 +MAX_OBJECT_BYTES=134217728 MAX_TOTAL_BYTES=2147483648 diff --git a/.gitignore b/.gitignore index 192ff04..b6462fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ out/ .env +.env.cluster +*.local.env data/ *.log +__pycache__/ +*.pyc +CLUSTER_PLAN.md +SCALE_OUT_PLAN.md diff --git a/Dockerfile b/Dockerfile index ac5b385..ea4898e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,22 @@ FROM eclipse-temurin:21-jdk-alpine AS build WORKDIR /src +RUN wget -q -O /tmp/postgresql.jar https://jdbc.postgresql.org/download/postgresql-42.7.14.jar && \ + echo '73914527305a40cce504b0d3d90b23caf565136912d607ae8ae7c5895512332c /tmp/postgresql.jar' | sha256sum -c - COPY src ./src COPY test ./test RUN mkdir /out && javac --release 21 --add-modules jdk.httpserver,java.net.http -d /out src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.StoreTest +RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.ConcurrencyTest RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.HttpTest +RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.ClusterNodeTest +RUN java -cp /out cloud.lunarsky.store.CliTest FROM eclipse-temurin:21-jre-alpine RUN addgroup -g 10001 store && adduser -D -u 10001 -G store store && mkdir /data && chown store:store /data COPY --from=build /out /app +COPY --from=build /tmp/postgresql.jar /app/postgresql.jar +COPY scripts/objectstore /usr/local/bin/objectstore +RUN chmod 755 /usr/local/bin/objectstore USER store EXPOSE 9000 -ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver", "-cp", "/app", "cloud.lunarsky.store.Main"] +ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.Main"] diff --git a/README.md b/README.md index 603ae5e..285fb4b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,58 @@ +![LunarSky ObjectStore banner](assets/objectstore-banner.svg) + # ObjectStore -Small S3-compatible object storage for LunarSky. Early development. +Small S3-compatible object storage for LunarSky. + +Source: [GitHub](https://github.com/LunarSkyOSS/ObjectStore) · [Gitea mirror](https://git.lunarsky.cloud/admin/ObjectStore) + +**Development software. Do not use it for production data.** It is provided as is, without warranty, under the [MIT License](LICENSE). + +[![Stage: development](assets/badge-stage.svg)](#limits-and-safety) [![License: MIT](assets/badge-license.svg)](LICENSE) [![Runtime: Java 21](assets/badge-java.svg)](Dockerfile) [![S3 API: partial support](assets/badge-api.svg)](#s3-api-support-checklist) + +## Contents + +- [Capability checklist](#capability-checklist) +- [S3 API support checklist](#s3-api-support-checklist) +- [Tests](TESTS.md) +- [Single-node setup](#single-node-setup) +- [CLI and tests](#cli-and-tests) +- [Local cluster prototype](#local-cluster-prototype) +- [Migrating a local cluster](#migrating-a-local-cluster) +- [Adding a cluster node](#adding-a-cluster-node) +- [Limits and safety](#limits-and-safety) +- [Disclaimer](#disclaimer) +- [AI contributions](#ai-contributions) + +## Capability checklist + +ObjectStore serves one configured bucket. + +- ✅ Persistent single-node storage with checksum verification +- ✅ Configurable per-object and total logical size limits +- ✅ CLI status, version, and full payload verification +- ✅ Local cluster prototype with stable node IDs and host-aware placement code +- ⬜ Automatic repair, rebalance, and garbage collection +- ⬜ Production multi-server deployment and metadata failover + +New objects retain their content type and key. Objects written by the earlier single-node format remain readable, but cannot appear in listings until overwritten because their original keys were not stored. + +## S3 API support checklist + +- ✅ Header-based AWS Signature Version 4 authentication +- ✅ `PutObject`, `GetObject`, `HeadObject`, and `DeleteObject` in both modes +- ✅ Single-range GET and `ListObjectsV2` in both modes +- ✅ SHA-256 payload verification and `x-amz-checksum-sha256` in both modes +- ✅ `CreateMultipartUpload`, `UploadPart`, `CompleteMultipartUpload`, and `AbortMultipartUpload` in single-node mode +- ⬜ Multipart uploads in cluster mode +- ⬜ Presigned URLs and streaming Signature V4 uploads +- ⬜ `CopyObject`, `ListParts`, and `ListMultipartUploads` +- ⬜ `Content-MD5` and checksum algorithms other than SHA-256 +- ⬜ Bucket creation and listing, object versioning, ACLs, tags, and user metadata + +This is an S3 API subset, not full AWS S3 compatibility. Unsupported S3 operations and Amazon-specific headers are rejected. + +## Single-node setup Requires Docker Compose. Copy `.env.example` to `.env`, then set `S3_ACCESS_KEY` and `S3_SECRET_KEY` to unique values. The access key must be at least 16 alphanumeric characters; the secret must be at least 32 characters. @@ -12,6 +64,64 @@ curl http://127.0.0.1:9000/health Port 9000 binds to localhost. Data stays in the `object-data` Docker volume. `docker compose down -v` deletes that volume. -Run `sh scripts/test.sh` with JDK 21 to test from source. Lunaris uses ObjectStore through its S3 storage adapter. +The standalone defaults are 128 MiB per object and 2 GiB total. Set `MAX_OBJECT_BYTES` and `MAX_TOTAL_BYTES` in `.env` to change them. Incomplete multipart uploads consume space until aborted. -Do not use this development version as the only copy of important data. +## CLI and tests + +From the server shell, run the CLI inside the running container from the directory containing `compose.yaml`: + +```sh +docker compose exec objectstore objectstore status +docker compose exec objectstore objectstore verify +docker compose exec objectstore objectstore version +``` + +The startup log shows the LunarSky banner, version, and a small storage summary (`docker compose logs --tail=20 objectstore`). `status` reports object and multipart usage. `verify` also checks stored payload hashes and exits nonzero on an error. Both commands can run while the service is live; they are not a snapshot or a backup. + +Run `sh scripts/test.sh` with JDK 21 to test from source. See [TESTS.md](TESTS.md) for coverage, the disposable Docker cluster suite, and the limits of those tests. + +## Local cluster prototype + +The local cluster prototype starts three segment containers and one PostgreSQL container on the same Docker host. Copy `.env.cluster.example` to a private environment file, replace all four credentials, and run: + +```sh +docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml up -d --build +sh scripts/test-cluster.sh /path/to/cluster.env +docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml run --rm repair +``` + +The cluster S3 endpoint binds to `127.0.0.1:9001`; storage nodes and PostgreSQL have no published ports. The separate repair container holds the repair credential and restores missing or corrupt replicas. + +Node UUIDs persist on their volumes, and replica manifests use those UUIDs so reordering configured URLs cannot move an existing replica. Each node also has an operator-assigned physical host UUID. New writes require acknowledgements from two different host UUIDs. The optional `CLUSTER_TEST_NODE_DOMAINS=true` override counts containers instead, solely for local process tests; all containers in this Compose file share one physical host. + +## Migrating a local cluster + +For an existing **local** three-node cluster that stores replicas by URL position: + +1. Stop the old gateway. Back up PostgreSQL and every node volume, and record the original ordered `CLUSTER_NODES`. +2. Start the nodes with the new image and a separate `CLUSTER_REPAIR_TOKEN`. Do not start the new gateway yet. +3. Run `objectstore cluster-migrate --check` in a one-off gateway container. Compare `legacy_0` through `legacy_2` and their URLs with the pre-upgrade inventory. +4. Run `objectstore cluster-migrate --apply id0,id1,id2` with those node IDs in the original order. The command verifies every listed live replica before conversion. +5. Start the new gateway only after the command reports `cluster_format=2`. Do not restart an old gateway against the converted database. + +The old format did not record the original URL mapping, so the inventory check is essential. The isolated migration fixture tests conversion; building this code does not automatically upgrade the running local prototype. + +## Adding a cluster node + +`objectstore cluster-join http://new-node:9100 expected-host-uuid` registers an additional local node. Add its URL to `CLUSTER_NODES` and restart the gateway to use it for new writes. Existing segments stay where their manifests say; this is capacity expansion for new writes, not a rebalance. `scripts/test-cluster.sh` exercises a fourth container joining and receiving new segments. + +## Limits and safety + +The cluster retains old and failed-write segments. It has no garbage collection, metadata standby, automated rebalance, private-network TLS, scoped credentials, or physical host verification yet. Host UUIDs are operator labels, not proof that machines have separate power, disks, or network paths. Keep `CLUSTER_LOCAL_DEV=true` limited to local tests. + +The standalone cluster node binds to localhost by default. Set `NODE_BIND` only for a private test network; the Compose file binds inside its private Docker network. PostgreSQL JDBC 42.7.14 is bundled in the image with its license inside the JAR. + +## Disclaimer + +ObjectStore is independent software. It is not affiliated with, sponsored by, or endorsed by Amazon or Amazon Web Services (AWS). “S3-compatible” describes only the API subset listed above. + +Keep independent backups; data loss is possible. The software is provided “as is” under the [MIT License](LICENSE). To the extent permitted by applicable law, the authors and maintainers are not responsible for data loss or other damages arising from its use. + +## AI contributions + +AI tools assisted with parts of this project. Maintainers review releases and remain responsible for what ships. diff --git a/TESTS.md b/TESTS.md new file mode 100644 index 0000000..2aa1f66 --- /dev/null +++ b/TESTS.md @@ -0,0 +1,60 @@ +# Testing ObjectStore + +ObjectStore is development software. Do not use it for production data. It is provided **as is, without warranty** under the [MIT License](LICENSE). Keep independent backups. Passing these tests does not guarantee that data cannot be lost. + +Run the commands below from the repository root. + +## Source tests + +Requires JDK 21. No Docker service is needed. + +```sh +sh scripts/test.sh +``` + +The script compiles the source and test programs into `out/classes`, then runs: + +| Test | Checks | +| --- | --- | +| `StoreTest` | Signature V4 test vector and tampering, local writes and reads, quotas, restart persistence, multipart recovery, legacy reads, locking, and corruption rejection. | +| `ConcurrencyTest` | Atomic local overwrites and consistent reads, listings, and deletes during concurrent access. | +| `HttpTest` | Signed HTTP requests, object operations, ranges, listing, and single-node multipart uploads. | +| `ClusterNodeTest` | Node identity and locking, authenticated segment transfers, checksum rejection, repair authorization, and restart cleanup. | +| `CliTest` | Version, status, verification, and a nonzero result for corrupt data. | + +The script exits nonzero on failure. The test programs use temporary local directories and loopback HTTP ports; they do not use an existing ObjectStore volume. + +## Disposable Docker cluster tests + +Requires Docker with Compose, Python 3, `curl`, and a free local port 9001. Make a test-only environment file from `.env.cluster.example` and fill in all five blank credentials with test-only values. Keep that file private and out of Git. + +```sh +cp .env.cluster.example /tmp/objectstore-cluster-tests.env +chmod 600 /tmp/objectstore-cluster-tests.env +``` + +After filling in the file, run: + +```sh +COMPOSE_PROJECT_NAME=objectstore-tests sh scripts/test-cluster.sh /tmp/objectstore-cluster-tests.env +``` + +Use a fresh, disposable Compose project. The script writes test objects, stops and restarts storage nodes and PostgreSQL, corrupts a replica to exercise repair, and joins a fourth node. It leaves the test stack running. To remove **only that test project's** containers and volumes after review: + +```sh +COMPOSE_PROJECT_NAME=objectstore-tests docker compose --env-file /tmp/objectstore-cluster-tests.env -f compose.cluster.yaml --profile expansion down -v +``` + +If port 9001 is occupied, set `CLUSTER_HOST_PORT` to the same free port in both the environment file and the shell before running the script. The script reads that port from the shell; Compose reads it from the file. + +The Docker suite checks signed S3 operations, multi-segment objects, concurrent overwrites, reads and writes with a node stopped, refusal to write without a storage quorum, restart recovery, corrupt-replica repair, metadata unavailability, and placement on a newly joined node. It also checks that containers labeled as one physical host cannot satisfy the normal host quorum. Its local-only override permits the remaining phases to use containers as separate test domains. + +`ClusterMigrationTest` is a separate legacy-format fixture and is **not** run by either test script. Do not run its `create` phase against a populated metadata database. The migration procedure is in the [README](README.md#migrating-a-local-cluster). + +## What these tests do not prove + +- Container stops are not physical power cuts or disk failures. The automated suite does not reboot a host or test every possible crash point. +- The Compose nodes share one machine. Passing the local-only quorum override does not demonstrate durability across independent hosts, racks, or sites. +- The suite does not test metadata failover, an off-site backup restore, prolonged load, or full AWS S3 compatibility. + +See [Limits and safety](README.md#limits-and-safety) before evaluating any multi-server deployment. diff --git a/assets/badge-api.svg b/assets/badge-api.svg new file mode 100644 index 0000000..f3a1482 --- /dev/null +++ b/assets/badge-api.svg @@ -0,0 +1,9 @@ + + + + + + + S3 API + PARTIAL + diff --git a/assets/badge-java.svg b/assets/badge-java.svg new file mode 100644 index 0000000..90ee8fa --- /dev/null +++ b/assets/badge-java.svg @@ -0,0 +1,9 @@ + + + + + + + RUNTIME + JAVA 21 + diff --git a/assets/badge-license.svg b/assets/badge-license.svg new file mode 100644 index 0000000..e982d6e --- /dev/null +++ b/assets/badge-license.svg @@ -0,0 +1,9 @@ + + + + + + + LICENSE + MIT + diff --git a/assets/badge-stage.svg b/assets/badge-stage.svg new file mode 100644 index 0000000..aa1d812 --- /dev/null +++ b/assets/badge-stage.svg @@ -0,0 +1,9 @@ + + + + + + + STAGE + DEVELOPMENT + diff --git a/assets/objectstore-banner.svg b/assets/objectstore-banner.svg new file mode 100644 index 0000000..6c5d17c --- /dev/null +++ b/assets/objectstore-banner.svg @@ -0,0 +1,61 @@ + + LunarSky ObjectStore + LunarSky woven hexagon logo and ObjectStore title on white with a soft prismatic gradient. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + LunarSky + + ObjectStore + diff --git a/compose.cluster.yaml b/compose.cluster.yaml new file mode 100644 index 0000000..60a0ba3 --- /dev/null +++ b/compose.cluster.yaml @@ -0,0 +1,127 @@ +services: + gateway: + build: . + image: lunarsky-objectstore:cluster-local + restart: unless-stopped + environment: + STORE_MODE: cluster + CLUSTER_LOCAL_DEV: "true" + CLUSTER_TEST_NODE_DOMAINS: "true" + BIND_ADDRESS: 0.0.0.0 + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + S3_BUCKET: ${S3_BUCKET:-objects} + S3_REGION: ${S3_REGION:-us-east-1} + MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728} + MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648} + CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN} + CLUSTER_NODES: ${CLUSTER_NODES:-http://node-a:9100,http://node-b:9100,http://node-c:9100} + POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10 + POSTGRES_USER: objectstore + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + ports: + - "127.0.0.1:${CLUSTER_HOST_PORT:-9001}:9000" + depends_on: + metadata: + condition: service_healthy + node-a: + condition: service_healthy + node-b: + condition: service_healthy + node-c: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9000/ready"] + interval: 10s + timeout: 4s + retries: 3 + mem_limit: 384m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + + metadata: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_DB: objectstore + POSTGRES_USER: objectstore + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + volumes: + - cluster-metadata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U objectstore -d objectstore"] + interval: 5s + timeout: 3s + retries: 10 + mem_limit: 256m + security_opt: + - no-new-privileges:true + + repair: + image: lunarsky-objectstore:cluster-local + profiles: [maintenance] + entrypoint: ["/usr/local/bin/objectstore", "cluster-repair"] + environment: + STORE_MODE: cluster + CLUSTER_LOCAL_DEV: "true" + CLUSTER_TEST_NODE_DOMAINS: "true" + CLUSTER_NODES: ${CLUSTER_NODES:-http://node-a:9100,http://node-b:9100,http://node-c:9100} + CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN} + CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN} + S3_BUCKET: ${S3_BUCKET:-objects} + POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10 + POSTGRES_USER: objectstore + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + mem_limit: 384m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + + node-a: &node + image: lunarsky-objectstore:cluster-local + restart: unless-stopped + entrypoint: ["java", "-XX:MaxRAMPercentage=70", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.ClusterNode"] + environment: + CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN} + CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN} + CLUSTER_HOST_ID: 5f1447b5-3f9e-457b-8ee7-e26f0c475b5f + DATA_DIR: /data + NODE_BIND: 0.0.0.0 + volumes: + - cluster-node-a:/data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/health"] + interval: 10s + timeout: 3s + retries: 3 + mem_limit: 256m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + + node-b: + <<: *node + volumes: + - cluster-node-b:/data + + node-c: + <<: *node + volumes: + - cluster-node-c:/data + + node-d: + <<: *node + profiles: [expansion] + volumes: + - cluster-node-d:/data + +volumes: + cluster-metadata: + cluster-node-a: + cluster-node-b: + cluster-node-c: + cluster-node-d: diff --git a/compose.yaml b/compose.yaml index b7362c2..f3053d2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,7 +8,7 @@ services: S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env} S3_BUCKET: ${S3_BUCKET:-objects} S3_REGION: ${S3_REGION:-us-east-1} - MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-10485760} + MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728} MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648} ports: - "127.0.0.1:${HOST_PORT:-9000}:9000" diff --git a/scripts/objectstore b/scripts/objectstore new file mode 100644 index 0000000..fed40ca --- /dev/null +++ b/scripts/objectstore @@ -0,0 +1,14 @@ +#!/bin/sh +if [ "${1:-}" = "cluster-repair" ]; then + shift + exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterRepair "$@" +fi +if [ "${1:-}" = "cluster-migrate" ]; then + shift + exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterMigrate "$@" +fi +if [ "${1:-}" = "cluster-join" ]; then + shift + exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterJoin "$@" +fi +exec java -XX:MaxRAMPercentage=70 -cp /app cloud.lunarsky.store.Cli "$@" diff --git a/scripts/test-cluster-http.py b/scripts/test-cluster-http.py new file mode 100644 index 0000000..43f8a70 --- /dev/null +++ b/scripts/test-cluster-http.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import datetime +import hashlib +import hmac +import pathlib +import sys +import urllib.error +import urllib.parse +import urllib.request + + +values = dict(line.strip().split("=", 1) for line in pathlib.Path(sys.argv[1]).read_text().splitlines() + if line.strip() and not line.startswith("#")) +access = values["S3_ACCESS_KEY"] +secret = values["S3_SECRET_KEY"] +bucket = values.get("S3_BUCKET", "objects") +port = values.get("CLUSTER_HOST_PORT", "9001") +host = f"127.0.0.1:{port}" + + +def sign(key, message): + return hmac.new(key, message.encode(), hashlib.sha256).digest() + + +def request(method, path, body=b"", extra=None): + extra = extra or {} + date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + stamp = date[:8] + digest = hashlib.sha256(body).hexdigest() + uri, _, query = path.partition("?") + canonical = "&".join(f"{urllib.parse.quote(k, safe='~-._')}={urllib.parse.quote(v, safe='~-._')}" + for k, v in sorted(urllib.parse.parse_qsl(query, keep_blank_values=True))) + headers = {"host": host, "x-amz-content-sha256": digest, "x-amz-date": date, **extra} + signed_names = ";".join(sorted(headers)) + canonical_headers = "".join(f"{name}:{headers[name]}\n" for name in sorted(headers)) + canonical_request = f"{method}\n{uri}\n{canonical}\n{canonical_headers}\n{signed_names}\n{digest}" + scope = f"{stamp}/us-east-1/s3/aws4_request" + to_sign = f"AWS4-HMAC-SHA256\n{date}\n{scope}\n{hashlib.sha256(canonical_request.encode()).hexdigest()}" + key = sign(sign(sign(sign(("AWS4" + secret).encode(), stamp), "us-east-1"), "s3"), "aws4_request") + signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() + headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={access}/{scope}," + f"SignedHeaders={signed_names},Signature={signature}") + url = f"http://{host}{path}" + outgoing = urllib.request.Request(url, data=body if method == "PUT" else None, + method=method, headers=headers) + try: + with urllib.request.urlopen(outgoing, timeout=30) as response: + return response.status, response.read(), response.headers + except urllib.error.HTTPError as error: + return error.code, error.read(), error.headers + + +if len(sys.argv) > 2 and sys.argv[2] == "survivor": + status, content, _ = request("GET", f"/{bucket}/cluster-test/survivor") + assert status == 200 and content == b"acknowledged object survives node loss", (status, content) + print("Cluster surviving-replica HTTP read passed") + sys.exit(0) + +if len(sys.argv) > 4 and sys.argv[2] == "status": + key = urllib.parse.quote(sys.argv[3], safe="/") + expected = int(sys.argv[4]) + status, _, _ = request("GET", f"/{bucket}/{key}") + assert status == expected, (status, expected) + print(f"Cluster GET status passed: {status}") + sys.exit(0) + + +key = f"/{bucket}/cluster-test/http.txt" +body = b"HTTP gateway integration test" +status, _, _ = request("PUT", key, body) +assert status == 200, status +status, content, _ = request("GET", key) +assert status == 200 and content == body, (status, content) +status, content, _ = request("GET", key, extra={"range": "bytes=5-11"}) +assert status == 206 and content == body[5:12], (status, content) +status, content, _ = request("HEAD", key) +assert status == 200 and not content, status +status, content, _ = request("GET", f"/{bucket}?list-type=2&prefix=cluster-test%2F") +assert status == 200 and b"cluster-test/http.txt" in content, (status, content) +status, _, _ = request("DELETE", key) +assert status == 204, status +status, _, _ = request("GET", key) +assert status == 404, status +print("Cluster HTTP tests passed: signed PUT, GET, range, HEAD, LIST, DELETE") diff --git a/scripts/test-cluster.sh b/scripts/test-cluster.sh new file mode 100644 index 0000000..2f077b2 --- /dev/null +++ b/scripts/test-cluster.sh @@ -0,0 +1,58 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +env_file=${1:?Usage: sh scripts/test-cluster.sh /path/to/local-cluster.env} +host_port=${CLUSTER_HOST_PORT:-9001} +compose() { docker compose --env-file "$env_file" -f compose.cluster.yaml "$@"; } +restore() { compose start metadata node-a node-b >/dev/null 2>&1 || true; } +trap restore EXIT +compose up -d --build +run_phase() { + compose exec -T gateway java --add-modules jdk.httpserver,java.net.http \ + -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterIntegrationTest "$1" +} +wait_ready() { + attempt=0 + until curl -fsS -o /dev/null "http://127.0.0.1:$host_port/ready" 2>/dev/null; do + attempt=$((attempt + 1)) + [ "$attempt" -lt 30 ] || return 1 + sleep 1 + done +} +run_phase basic +run_phase same-host +run_phase concurrent +compose stop node-a +run_phase degraded +compose stop node-b +run_phase quorum-lost +compose start node-a node-b +wait_ready +run_phase recovered +segment_id=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \ + "SELECT s.segment_id FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE o.object_key='cluster-test/survivor' LIMIT 1") +printf '%s\n' "$segment_id" | grep -Eq '^[0-9a-f-]{36}$' +shard=$(printf '%s' "$segment_id" | cut -c1-2) +compose exec -T node-a sh -c 'printf corrupted > "/data/segments/$1/$2"' _ "$shard" "$segment_id" +run_phase recovered +compose run --rm -T repair +expected=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \ + "SELECT encode(s.sha256,'hex') FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE o.object_key='cluster-test/survivor' LIMIT 1") +actual=$(compose exec -T node-a sha256sum "/data/segments/$shard/$segment_id" | cut -d' ' -f1) +[ "$expected" = "$actual" ] +compose stop metadata +status=$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:$host_port/ready") +[ "$status" = 503 ] +compose start metadata +wait_ready +run_phase recovered +python3 scripts/test-cluster-http.py "$env_file" +compose --profile expansion up -d node-d +compose run --rm -T --no-deps --entrypoint /usr/local/bin/objectstore gateway \ + cluster-join http://node-d:9100 5f1447b5-3f9e-457b-8ee7-e26f0c475b5f +export CLUSTER_NODES=http://node-a:9100,http://node-b:9100,http://node-c:9100,http://node-d:9100 +compose up -d --no-deps gateway +wait_ready +run_phase recovered +run_phase joined +echo 'Cluster failure tests passed' diff --git a/scripts/test.sh b/scripts/test.sh index 9703feb..adb41e3 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -5,4 +5,7 @@ mkdir -p out/classes javac --release 21 --add-modules jdk.httpserver,java.net.http -d out/classes \ src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.StoreTest +java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.ConcurrencyTest java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.HttpTest +java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.ClusterNodeTest +java -cp out/classes cloud.lunarsky.store.CliTest diff --git a/src/cloud/lunarsky/store/Cli.java b/src/cloud/lunarsky/store/Cli.java new file mode 100644 index 0000000..dc690ec --- /dev/null +++ b/src/cloud/lunarsky/store/Cli.java @@ -0,0 +1,154 @@ +package cloud.lunarsky.store; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.List; + +public final class Cli { + private Cli() {} + + static final class Report { + long objects, legacyObjects, payloadBytes, recordBytes, uploads, stagedBytes, checked, changedDuringScan; + long errors; + final List problems = new ArrayList<>(); + + void problem(String message) { + errors++; + if (problems.size() < 100) problems.add(message); + } + } + + public static void main(String[] args) { + int result = run(args, Path.of(System.getenv().getOrDefault("DATA_DIR", "/data")), System.out, System.err); + if (result != 0) System.exit(result); + } + + static int run(String[] args, Path root, PrintStream out, PrintStream err) { + if (args.length == 1 && (args[0].equals("version") || args[0].equals("--version"))) { + out.println("ObjectStore " + Version.VALUE); + return 0; + } + if (args.length == 1 && (args[0].equals("help") || args[0].equals("--help"))) { + out.println("Usage: objectstore status|verify|version"); + out.println("status Show stored object and multipart usage"); + out.println("verify Check object records, paths and payload checksums"); + out.println("version Show the ObjectStore version"); + return 0; + } + if (args.length != 1 || !(args[0].equals("status") || args[0].equals("verify"))) { + err.println("Usage: objectstore status|verify|version"); + return 2; + } + try { + boolean verify = args[0].equals("verify"); + Report report = inspect(root, verify); + out.println("version=" + Version.VALUE); + out.println("objects=" + report.objects); + out.println("legacy_objects=" + report.legacyObjects); + out.println("payload_bytes=" + report.payloadBytes); + out.println("record_bytes=" + report.recordBytes); + out.println("multipart_uploads=" + report.uploads); + out.println("multipart_staged_bytes=" + report.stagedBytes); + if (verify) out.println("verified_objects=" + report.checked); + if (report.changedDuringScan > 0) out.println("changed_during_scan=" + report.changedDuringScan); + out.println("errors=" + report.errors); + for (String problem : report.problems) err.println(problem); + if (report.errors > report.problems.size()) + err.println((report.errors - report.problems.size()) + " further errors omitted"); + return report.errors == 0 ? 0 : 1; + } catch (IOException error) { + err.println("ObjectStore inspection failed: " + error.getMessage()); + return 1; + } + } + + static Report inspect(Path root, boolean verify) throws IOException { + Path objects = root.resolve("objects"); + if (!Files.isDirectory(objects)) throw new IOException("Object data directory does not exist: " + objects); + Report report = new Report(); + try (var paths = Files.walk(objects)) { + for (Path path : paths.filter(Files::isRegularFile).toList()) inspectObject(objects, path, verify, report); + } + Path multipart = root.resolve("multipart"); + if (Files.isDirectory(multipart)) { + try (var uploads = Files.list(multipart)) { + for (Path dir : uploads.toList()) inspectUpload(dir, report); + } + } + return report; + } + + private static void inspectObject(Path objects, Path path, boolean verify, Report report) { + try (var channel = FileChannel.open(path, StandardOpenOption.READ); + var input = new DataInputStream(Channels.newInputStream(channel))) { + long size = channel.size(); + var record = DiskStore.readRecord(input); + var meta = record.metadata(); + report.objects++; + report.payloadBytes += meta.length(); + report.recordBytes += size; + if (meta.key() == null) report.legacyObjects++; + else { + String id = SigV4.hex(SigV4.hash((meta.bucket() + "/" + meta.key()).getBytes(StandardCharsets.UTF_8))); + Path expected = objects.resolve(id.substring(0, 2)).resolve(id); + if (!path.equals(expected)) report.problem("Mismatched object path: " + path); + } + if (size - record.headerLength() != meta.length()) { + report.problem("Invalid object length: " + path); + return; + } + if (verify) { + MessageDigest sha = digest("SHA-256"), md5 = digest("MD5"); + byte[] buffer = new byte[65536]; long count = 0; int n; + while ((n = input.read(buffer)) != -1) { + count += n; + sha.update(buffer, 0, n); + md5.update(buffer, 0, n); + } + report.checked++; + if (count != meta.length() || !MessageDigest.isEqual(sha.digest(), meta.sha256()) || + !SigV4.hex(md5.digest()).equals(meta.etag())) + report.problem("Object checksum mismatch: " + path); + } + } catch (NoSuchFileException error) { + report.changedDuringScan++; + } catch (IOException | RuntimeException error) { + report.problem("Unreadable object record: " + path + " (" + error.getClass().getSimpleName() + ")"); + } + } + + private static void inspectUpload(Path dir, Report report) throws IOException { + if (!Files.isDirectory(dir)) { + report.problem("Unexpected multipart entry: " + dir); + return; + } + report.uploads++; + if (!Files.isRegularFile(dir.resolve("manifest"))) report.problem("Missing multipart manifest: " + dir); + try (var files = Files.list(dir)) { + for (Path path : files.toList()) { + String name = path.getFileName().toString(); + if (name.matches("part-[0-9]{5}")) { + try { report.stagedBytes += Files.size(path); } + catch (NoSuchFileException error) { report.changedDuringScan++; } + } else if (!name.equals("manifest")) report.problem("Unexpected multipart file: " + path); + } + } catch (NoSuchFileException error) { + report.changedDuringScan++; + } + } + + private static MessageDigest digest(String name) { + try { return MessageDigest.getInstance(name); } + catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); } + } +} diff --git a/src/cloud/lunarsky/store/ClusterJoin.java b/src/cloud/lunarsky/store/ClusterJoin.java new file mode 100644 index 0000000..7fc8e03 --- /dev/null +++ b/src/cloud/lunarsky/store/ClusterJoin.java @@ -0,0 +1,29 @@ +package cloud.lunarsky.store; + +import java.net.URI; +import java.sql.DriverManager; +import java.util.Map; +import java.util.UUID; + +public final class ClusterJoin { + private ClusterJoin() {} + + public static void main(String[] args) throws Exception { + if (args.length != 2) + throw new IllegalArgumentException("Usage: objectstore cluster-join node-url expected-host-uuid"); + Map env = System.getenv(); + if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV"))) + throw new IllegalArgumentException("Node registration is only enabled in local cluster mode"); + URI url = URI.create(args[0]); + UUID host = UUID.fromString(args[1]); + try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"), + env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"))) { + if (SchemaMigrator.prepare(connection, env.get("S3_BUCKET")) != 2) + throw new IllegalStateException("Migrate legacy replicas before joining nodes"); + NodeClient.Node node = NodeRegistry.join(connection, url, host, env.get("CLUSTER_TOKEN")); + System.out.println("node_id=" + node.id()); + System.out.println("host_id=" + node.hostId()); + System.out.println("endpoint=" + node.url()); + } + } +} diff --git a/src/cloud/lunarsky/store/ClusterMigrate.java b/src/cloud/lunarsky/store/ClusterMigrate.java new file mode 100644 index 0000000..c8ace99 --- /dev/null +++ b/src/cloud/lunarsky/store/ClusterMigrate.java @@ -0,0 +1,157 @@ +package cloud.lunarsky.store; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public final class ClusterMigrate { + private ClusterMigrate() {} + + public static void main(String[] args) throws Exception { + if ((args.length != 1 || !args[0].equals("--check")) && + (args.length != 2 || !args[0].equals("--apply"))) + throw new IllegalArgumentException("Usage: objectstore cluster-migrate --check | --apply node-id-0,node-id-1,node-id-2"); + Map env = System.getenv(); + if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV"))) + throw new IllegalArgumentException("Migration is enabled only in local cluster mode"); + List urls = Arrays.stream(env.get("CLUSTER_NODES").split(",", -1)).map(URI::create).toList(); + if (urls.size() != 3) throw new IllegalArgumentException("Legacy migration requires the original three URLs in their original order"); + String token = env.get("CLUSTER_TOKEN"); + List addresses = new ArrayList<>(); + for (URI url : urls) { + NodeIdentity identity = NodeClient.probe(url, token); + addresses.add(new NodeClient.Node(identity.nodeId(), identity.hostId(), url)); + } + NodeClient nodes = new NodeClient(addresses, token, null); + if (args[0].equals("--apply")) { + String actual = addresses.stream().map(node -> node.id().toString()) + .collect(java.util.stream.Collectors.joining(",")); + if (!actual.equals(args[1])) + throw new IllegalArgumentException("Confirmed legacy node mapping differs from the current ordered node identities"); + } + try (Connection connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"), + env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"))) { + int format = SchemaMigrator.prepare(connection, env.get("S3_BUCKET")); + System.out.println("cluster_format=" + format); + if (format == 2) return; + for (int index = 0; index < nodes.count(); index++) { + NodeClient.Node node = nodes.node(index); + System.out.println("legacy_" + index + "=" + node.id() + " host=" + node.hostId() + + " endpoint=" + node.url()); + } + long verified = verifyLiveSegments(connection, nodes); + System.out.println("live_segments_verified=" + verified); + if (args[0].equals("--check")) return; + apply(connection, nodes); + System.out.println("cluster_format=2"); + } + } + + private static long verifyLiveSegments(Connection connection, NodeClient nodes) throws SQLException, IOException { + long verified = 0; + try (PreparedStatement query = connection.prepareStatement( + "SELECT s.segment_id, s.length, s.sha256, s.replicas FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation"); + ResultSet result = query.executeQuery()) { + while (result.next()) { + UUID segment = (UUID) result.getObject(1); + int length = result.getInt(2); + byte[] hash = result.getBytes(3); + for (int index : legacyIndices(result.getString(4), nodes.count())) { + try { + nodes.get(index, segment, length, hash); + } catch (IOException offlineOrCorrupt) { + throw new IOException("A listed live replica is unavailable or corrupt: segment " + + segment + " legacy node " + index, offlineOrCorrupt); + } + } + verified++; + } + } + return verified; + } + + private static List legacyIndices(String text, int count) throws IOException { + if (text == null || text.isBlank()) throw new IOException("Missing legacy replica list"); + List indices = new ArrayList<>(); + Set unique = new HashSet<>(); + for (String part : text.split(",", -1)) { + int index; + try { index = Integer.parseInt(part); } + catch (NumberFormatException error) { throw new IOException("Invalid legacy replica index", error); } + if (index < 0 || index >= count || !unique.add(index)) + throw new IOException("Invalid or duplicate legacy replica index"); + indices.add(index); + } + return indices; + } + + private static void apply(Connection connection, NodeClient nodes) throws SQLException, IOException { + try { + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.execute("SELECT pg_advisory_xact_lock(6834071092781)"); + try (ResultSet result = statement.executeQuery("SELECT version FROM cluster_format WHERE singleton=1 FOR UPDATE")) { + if (!result.next() || result.getInt(1) != 1) + throw new IOException("Cluster format changed during migration"); + } + try (ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_nodes)")) { + result.next(); + if (result.getBoolean(1)) throw new IOException("Legacy migration already has node bindings"); + } + } + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_nodes (node_id, host_id, endpoint, legacy_index, state) VALUES (?, ?, ?, ?, 'active')")) { + for (int index = 0; index < nodes.count(); index++) { + NodeClient.Node node = nodes.node(index); + insert.setObject(1, node.id()); + insert.setObject(2, node.hostId()); + insert.setString(3, node.url().toString()); + insert.setInt(4, index); + insert.addBatch(); + } + insert.executeBatch(); + } + try (PreparedStatement select = connection.prepareStatement( + "SELECT generation, ordinal, replicas FROM cluster_segments WHERE replica_ids IS NULL"); + ResultSet result = select.executeQuery(); + PreparedStatement update = connection.prepareStatement( + "UPDATE cluster_segments SET replica_ids=? WHERE generation=? AND ordinal=? AND replica_ids IS NULL")) { + while (result.next()) { + List ids = new ArrayList<>(); + for (int index : legacyIndices(result.getString(3), nodes.count())) + ids.add(nodes.node(index).id()); + update.setArray(1, connection.createArrayOf("uuid", ids.toArray())); + update.setObject(2, result.getObject(1)); + update.setInt(3, result.getInt(2)); + if (update.executeUpdate() != 1) throw new IOException("Segment changed during migration"); + } + } + try (Statement statement = connection.createStatement()) { + try (ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL)")) { + result.next(); + if (result.getBoolean(1)) throw new IOException("Unconverted legacy segments remain"); + } + statement.executeUpdate("UPDATE cluster_format SET version=2 WHERE singleton=1 AND version=1"); + statement.execute("ALTER TABLE cluster_segments VALIDATE CONSTRAINT cluster_replica_ids_required"); + } + connection.commit(); + } catch (SQLException | IOException | RuntimeException error) { + try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); } + if (error instanceof IOException io) throw io; + if (error instanceof SQLException sql) throw sql; + throw (RuntimeException) error; + } finally { connection.setAutoCommit(true); } + } +} diff --git a/src/cloud/lunarsky/store/ClusterNode.java b/src/cloud/lunarsky/store/ClusterNode.java new file mode 100644 index 0000000..2b2b90a --- /dev/null +++ b/src/cloud/lunarsky/store/ClusterNode.java @@ -0,0 +1,244 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Executors; + +/** Serves verified object segments to authenticated cluster peers. */ +public final class ClusterNode implements AutoCloseable { + static final int MAX_SEGMENT = 8 * 1024 * 1024; + private final Path root, segments, pending; + private final byte[] token; + private final byte[] repairToken; + private final NodeIdentity identity; + private final FileChannel lockChannel; + private final FileLock lock; + + ClusterNode(Path root, String token, String repairToken, UUID hostId) throws IOException { + if (token == null || token.length() < 32) throw new IllegalArgumentException("Cluster token must have at least 32 characters"); + if (repairToken == null || repairToken.length() < 32 || repairToken.equals(token)) + throw new IllegalArgumentException("A separate repair token of at least 32 characters is required"); + if (hostId == null) throw new IllegalArgumentException("Storage host ID is required"); + this.root = root; + this.token = token.getBytes(java.nio.charset.StandardCharsets.UTF_8); + this.repairToken = repairToken.getBytes(java.nio.charset.StandardCharsets.UTF_8); + segments = root.resolve("segments"); + pending = root.resolve("pending"); + Files.createDirectories(root); + lockChannel = FileChannel.open(root.resolve(".process.lock"), StandardOpenOption.CREATE, StandardOpenOption.WRITE); + FileLock acquired; + try { acquired = lockChannel.tryLock(); } + catch (OverlappingFileLockException error) { + lockChannel.close(); + throw new IOException("Node data directory is already in use", error); + } + if (acquired == null) { + lockChannel.close(); + throw new IOException("Node data directory is already in use"); + } + lock = acquired; + try { + identity = NodeIdentity.open(root, hostId); + Files.createDirectories(segments); + Files.createDirectories(pending); + DiskStore.syncDirectory(root); + try (var files = Files.list(pending)) { + for (Path file : files.toList()) { + if (!Files.isRegularFile(file)) throw new IOException("Invalid pending entry: " + file); + Files.delete(file); + } + } + DiskStore.syncDirectory(pending); + } catch (IOException error) { + close(); + throw error; + } + } + + void handle(HttpExchange exchange) throws IOException { + try { + String path = exchange.getRequestURI().getPath(); + if (path.equals("/health") && exchange.getRequestMethod().equals("GET")) { + respond(exchange, 200, "ok"); + return; + } + byte[] supplied = exchange.getRequestHeaders().getFirst("X-Cluster-Token") == null + ? new byte[0] : exchange.getRequestHeaders().getFirst("X-Cluster-Token") + .getBytes(java.nio.charset.StandardCharsets.UTF_8); + if (!MessageDigest.isEqual(token, supplied)) { + respond(exchange, 403, "Forbidden"); + return; + } + if (path.equals("/identity") && exchange.getRequestMethod().equals("GET")) { + respond(exchange, 200, identity.nodeId() + " " + identity.hostId()); + return; + } + if (!identity.nodeId().toString().equals(exchange.getRequestHeaders().getFirst("X-Cluster-Expected-Node"))) { + respond(exchange, 409, "Wrong storage node"); + return; + } + if (exchange.getRequestMethod().equals("PUT") && + "true".equals(exchange.getRequestHeaders().getFirst("X-Cluster-Repair"))) { + String suppliedRepair = exchange.getRequestHeaders().getFirst("X-Cluster-Repair-Token"); + byte[] suppliedBytes = suppliedRepair == null ? new byte[0] + : suppliedRepair.getBytes(java.nio.charset.StandardCharsets.UTF_8); + if (!MessageDigest.isEqual(repairToken, suppliedBytes)) { + respond(exchange, 403, "Repair authority required"); + return; + } + } + if (!path.matches("/segments/[0-9a-f-]{36}")) { + respond(exchange, 404, "Not found"); + return; + } + String id = path.substring("/segments/".length()); + if (!UUID.fromString(id).toString().equals(id)) { + respond(exchange, 400, "Invalid segment ID"); + return; + } + switch (exchange.getRequestMethod()) { + case "PUT" -> put(exchange, segmentPath(id, true)); + case "GET" -> get(exchange, segmentPath(id, false)); + default -> respond(exchange, 405, "Method not allowed"); + } + } catch (IllegalArgumentException error) { + respond(exchange, 400, "Invalid request"); + } catch (IOException error) { + if (exchange.getResponseCode() == -1) respond(exchange, 500, "Storage failure"); + throw error; + } finally { + exchange.close(); + } + } + + private synchronized Path segmentPath(String id, boolean createShard) throws IOException { + Path shard = segments.resolve(id.substring(0, 2)); + if (createShard && !Files.isDirectory(shard)) { + Files.createDirectories(shard); + DiskStore.syncDirectory(segments); + } + return shard.resolve(id); + } + + private void put(HttpExchange exchange, Path target) throws IOException { + String hash = exchange.getRequestHeaders().getFirst("X-Cluster-Sha256"); + String lengthText = exchange.getRequestHeaders().getFirst("Content-Length"); + if (hash == null || !hash.matches("[0-9a-f]{64}") || lengthText == null) { + respond(exchange, 400, "Missing checksum or length"); + return; + } + long length = Long.parseLong(lengthText); + if (length < 1 || length > MAX_SEGMENT) { + respond(exchange, 413, "Segment too large"); + return; + } + Path temp = Files.createTempFile(pending, "segment-", ".part"); + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + long count = 0; + try (InputStream input = exchange.getRequestBody(); OutputStream output = Files.newOutputStream(temp)) { + byte[] buffer = new byte[65536]; + int read; + while ((read = input.read(buffer)) != -1) { + count += read; + if (count > length) { + respond(exchange, 400, "Body longer than declared length"); + return; + } + digest.update(buffer, 0, read); + output.write(buffer, 0, read); + } + } + if (count != length || !hash.equals(HexFormat.of().formatHex(digest.digest()))) { + respond(exchange, 400, "Segment checksum mismatch"); + return; + } + try (FileChannel channel = FileChannel.open(temp, StandardOpenOption.WRITE)) { + channel.force(true); + } + synchronized (this) { + if (Files.exists(target)) { + byte[] existing = Files.readAllBytes(target); + if (!Arrays.equals(existing, Files.readAllBytes(temp))) { + if (!"true".equals(exchange.getRequestHeaders().getFirst("X-Cluster-Repair")) || + hash.equals(HexFormat.of().formatHex(SigV4.hash(existing)))) { + respond(exchange, 409, "Segment ID conflict"); + return; + } + Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + DiskStore.syncDirectory(target.getParent()); + } + } else { + Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE); + DiskStore.syncDirectory(target.getParent()); + } + } + respond(exchange, 200, "ok"); + } catch (java.security.NoSuchAlgorithmException error) { + throw new IllegalStateException(error); + } finally { + Files.deleteIfExists(temp); + } + } + + private void get(HttpExchange exchange, Path target) throws IOException { + if (!Files.isRegularFile(target)) { + respond(exchange, 404, "Segment not found"); + return; + } + long length = Files.size(target); + if (length > MAX_SEGMENT) throw new IOException("Segment exceeds maximum length"); + exchange.sendResponseHeaders(200, length); + try (InputStream input = Files.newInputStream(target)) { + input.transferTo(exchange.getResponseBody()); + } + } + + private static void respond(HttpExchange exchange, int status, String message) throws IOException { + byte[] body = message.getBytes(java.nio.charset.StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "text/plain; charset=utf-8"); + exchange.sendResponseHeaders(status, body.length); + exchange.getResponseBody().write(body); + } + + @Override public void close() throws IOException { + lock.release(); + lockChannel.close(); + } + + public static void main(String[] args) throws Exception { + Map env = System.getenv(); + String token = env.get("CLUSTER_TOKEN"); + var node = new ClusterNode(Path.of(env.getOrDefault("DATA_DIR", "/data")), token, + env.get("CLUSTER_REPAIR_TOKEN"), + UUID.fromString(env.get("CLUSTER_HOST_ID"))); + int port = Integer.parseInt(env.getOrDefault("NODE_PORT", "9100")); + var server = HttpServer.create(new InetSocketAddress(env.getOrDefault("NODE_BIND", "127.0.0.1"), port), 64); + var executor = Executors.newVirtualThreadPerTaskExecutor(); + server.setExecutor(executor); + server.createContext("/", node::handle); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + server.stop(5); + executor.close(); + try { node.close(); } catch (IOException error) { System.err.println("Node close failed: " + error); } + })); + server.start(); + System.out.println("ObjectStore cluster node listening on :" + port); + } +} diff --git a/src/cloud/lunarsky/store/ClusterRepair.java b/src/cloud/lunarsky/store/ClusterRepair.java new file mode 100644 index 0000000..1f4889a --- /dev/null +++ b/src/cloud/lunarsky/store/ClusterRepair.java @@ -0,0 +1,26 @@ +package cloud.lunarsky.store; + +import java.net.URI; +import java.util.Arrays; +import java.util.Map; + +public final class ClusterRepair { + public static void main(String[] args) throws Exception { + if (args.length != 0) throw new IllegalArgumentException("Usage: objectstore cluster-repair"); + Map env = System.getenv(); + if (!"cluster".equals(env.get("STORE_MODE")) || !"true".equals(env.get("CLUSTER_LOCAL_DEV"))) + throw new IllegalArgumentException("Cluster repair is only enabled in local cluster mode"); + try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"), + env.get("POSTGRES_PASSWORD"), env.get("S3_BUCKET"), + Arrays.stream(env.get("CLUSTER_NODES").split(",")).map(URI::create).toList(), + env.get("CLUSTER_TOKEN"), env.get("CLUSTER_REPAIR_TOKEN"), 134217728, 2147483648L, + "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS")))) { + var report = store.repairOnce(); + System.out.println("segments_scanned=" + report.scanned()); + System.out.println("replicas_restored=" + report.restored()); + System.out.println("segments_under_replicated=" + report.underReplicated()); + System.out.println("segments_unrecoverable=" + report.unrecoverable()); + if (report.unrecoverable() > 0) System.exit(1); + } + } +} diff --git a/src/cloud/lunarsky/store/ClusterStore.java b/src/cloud/lunarsky/store/ClusterStore.java new file mode 100644 index 0000000..843a241 --- /dev/null +++ b/src/cloud/lunarsky/store/ClusterStore.java @@ -0,0 +1,442 @@ +package cloud.lunarsky.store; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.DigestInputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HexFormat; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +final class ClusterStore implements ObjectStorage { + private record Segment(UUID id, int length, byte[] hash, List replicas) {} + private record RepairTarget(UUID generation, int ordinal, long version, Segment segment) {} + record RepairReport(int scanned, int restored, int underReplicated, int unrecoverable) {} + private final String jdbcUrl, user, password, configuredBucket; + private final NodeClient nodes; + private final long maxObject, maxTotal; + private final boolean testNodeDomains; + + ClusterStore(String jdbcUrl, String user, String password, String bucket, + List nodeUrls, String token, String repairToken, long maxObject, long maxTotal, + boolean testNodeDomains) throws IOException { + if (jdbcUrl == null || !jdbcUrl.startsWith("jdbc:postgresql://") || user == null || password == null) + throw new IllegalArgumentException("Invalid metadata database configuration"); + this.jdbcUrl = jdbcUrl; this.user = user; this.password = password; + this.configuredBucket = bucket; this.maxObject = maxObject; this.maxTotal = maxTotal; + this.testNodeDomains = testNodeDomains; + try (Connection connection = connect()) { + int format = SchemaMigrator.prepare(connection, bucket); + if (format != 2) throw new IOException("Legacy replica positions require objectstore cluster-migrate before this gateway can start"); + nodes = NodeRegistry.load(connection, nodeUrls, token, repairToken); + } catch (SQLException error) { throw databaseError(error); } + } + + private Connection connect() throws SQLException { return DriverManager.getConnection(jdbcUrl, user, password); } + + @Override public Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, + String checksum, boolean createOnly, String contentType) throws IOException { + if (!configuredBucket.equals(bucket)) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required"); + if (length > maxObject) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit"); + if (!nodes.availableHostsAtLeast(2, testNodeDomains)) + throw new StoreException(503, "SlowDown", "Fewer than two storage hosts are available"); + if (contentType.getBytes(java.nio.charset.StandardCharsets.UTF_8).length > 255) + throw new StoreException(400, "InvalidArgument", "Content-Type is too long"); + MessageDigest sha = digest("SHA-256"), md5 = digest("MD5"); + List segments = new ArrayList<>(); + byte[] fullHash; + Path staged = Files.createTempFile("objectstore-cluster-", ".pending"); + try { + try (OutputStream output = Files.newOutputStream(staged)) { + byte[] buffer = new byte[65536]; + long remaining = length; + while (remaining > 0) { + int count = input.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (count < 0) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length"); + if (count == 0) continue; + sha.update(buffer, 0, count); md5.update(buffer, 0, count); + output.write(buffer, 0, count); + remaining -= count; + } + } + if (input.read() != -1) throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size"); + fullHash = sha.digest(); + if (!HexFormat.of().formatHex(fullHash).equals(expectedHash)) + throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch"); + if (checksum != null && !Base64.getEncoder().encodeToString(fullHash).equals(checksum)) + throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch"); + try (Connection connection = connect()) { + long previous = currentLength(connection, bucket, key); + if (createOnly && previous >= 0) + throw new StoreException(412, "PreconditionFailed", "Object already exists"); + try (PreparedStatement query = connection.prepareStatement("SELECT used_bytes FROM cluster_usage WHERE bucket=?")) { + query.setString(1, bucket); + try (ResultSet result = query.executeQuery()) { + if (!result.next()) throw new SQLException("Bucket quota row is missing"); + if (result.getLong(1) - Math.max(0, previous) > maxTotal - length) + throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); + } + } + } catch (SQLException error) { throw databaseError(error); } + try (InputStream stagedInput = Files.newInputStream(staged)) { + long remaining = length; + while (remaining > 0) { + int wanted = (int) Math.min(ClusterNode.MAX_SEGMENT, remaining); + byte[] bytes = stagedInput.readNBytes(wanted); + if (bytes.length != wanted) throw new IOException("Staged object was truncated"); + byte[] segmentHash = SigV4.hash(bytes); + UUID id = UUID.randomUUID(); + List replicas = new ArrayList<>(); + Set acceptedHosts = new HashSet<>(); + for (int index : PlacementPolicy.candidates(id, nodes, testNodeDomains)) { + UUID host = nodes.faultDomain(index, testNodeDomains); + if (acceptedHosts.contains(host)) continue; + try { + nodes.put(index, id, bytes, segmentHash); + replicas.add(nodes.node(index).id()); + acceptedHosts.add(host); + if (acceptedHosts.size() == 3) break; + } catch (IOException error) { + System.err.println("Cluster node " + nodes.node(index).id() + + " did not accept segment " + id + ": " + error.getMessage()); + } + } + if (acceptedHosts.size() < 2) + throw new StoreException(503, "SlowDown", "Fewer than two storage hosts accepted the segment"); + segments.add(new Segment(id, wanted, segmentHash, List.copyOf(replicas))); + remaining -= wanted; + } + } + } finally { Files.deleteIfExists(staged); } + Metadata metadata = new Metadata(length, Instant.now().toEpochMilli(), + HexFormat.of().formatHex(md5.digest()), fullHash, bucket, key, contentType); + UUID generation = UUID.randomUUID(); + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try { + long used = lockUsage(connection, bucket); + long previous = currentLength(connection, bucket, key); + if (createOnly && previous >= 0) + throw new StoreException(412, "PreconditionFailed", "Object already exists"); + if (used - Math.max(0, previous) > maxTotal - length) + throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_segments (generation, ordinal, segment_id, length, sha256, replicas, replica_ids) VALUES (?, ?, ?, ?, ?, 'v2', ?)")) { + for (int i = 0; i < segments.size(); i++) { + Segment segment = segments.get(i); + insert.setObject(1, generation); insert.setInt(2, i); insert.setObject(3, segment.id()); + insert.setInt(4, segment.length()); insert.setBytes(5, segment.hash()); + insert.setArray(6, connection.createArrayOf("uuid", segment.replicas().toArray())); + insert.addBatch(); + } + insert.executeBatch(); + } + try (PreparedStatement update = connection.prepareStatement( + "INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, length=EXCLUDED.length, modified=EXCLUDED.modified, etag=EXCLUDED.etag, sha256=EXCLUDED.sha256, content_type=EXCLUDED.content_type")) { + bindObject(update, metadata, generation); update.executeUpdate(); + } + try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) { + update.setLong(1, used - Math.max(0, previous) + length); update.setString(2, bucket); update.executeUpdate(); + } + try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) { + delete.setString(1, bucket); delete.setString(2, key); delete.executeUpdate(); + } + connection.commit(); + return metadata; + } catch (SQLException | RuntimeException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public OpenObject open(String bucket, String key) throws IOException { + try (Connection connection = connect()) { + connection.setAutoCommit(false); + connection.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ); + try { + Metadata metadata; + UUID generation; + try (PreparedStatement query = connection.prepareStatement( + "SELECT generation, length, modified, etag, sha256, content_type FROM cluster_objects WHERE bucket=? AND object_key=?")) { + query.setString(1, bucket); query.setString(2, key); + try (ResultSet result = query.executeQuery()) { + if (!result.next()) throw new StoreException(404, "NoSuchKey", "Object not found"); + generation = (UUID) result.getObject(1); + metadata = new Metadata(result.getLong(2), result.getLong(3), result.getString(4), + result.getBytes(5), bucket, key, result.getString(6)); + } + } + List parts = new ArrayList<>(); + try (PreparedStatement query = connection.prepareStatement( + "SELECT segment_id, length, sha256, replica_ids FROM cluster_segments WHERE generation=? ORDER BY ordinal")) { + query.setObject(1, generation); + try (ResultSet result = query.executeQuery()) { + long total = 0; + while (result.next()) { + Segment segment = new Segment((UUID) result.getObject(1), result.getInt(2), + result.getBytes(3), replicaIds(result, 4)); + total = Math.addExact(total, segment.length()); + parts.add(segment); + } + if (total != metadata.length()) throw new IOException("Incomplete object manifest"); + } + } + connection.commit(); + return new OpenObject(metadata, verifiedObject(parts, metadata)); + } catch (SQLException | RuntimeException | IOException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + if (error instanceof IOException io) throw io; + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public void delete(String bucket, String key) throws IOException { + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try { + long used = lockUsage(connection, bucket); + long previous = currentLength(connection, bucket, key); + try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_objects WHERE bucket=? AND object_key=?")) { + delete.setString(1, bucket); delete.setString(2, key); delete.executeUpdate(); + } + try (PreparedStatement update = connection.prepareStatement( + "INSERT INTO cluster_tombstones VALUES (?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, deleted_at=EXCLUDED.deleted_at")) { + update.setString(1, bucket); update.setString(2, key); + update.setObject(3, UUID.randomUUID()); update.setLong(4, Instant.now().toEpochMilli()); + update.executeUpdate(); + } + if (previous >= 0) { + try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) { + update.setLong(1, used - previous); update.setString(2, bucket); update.executeUpdate(); + } + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) throws IOException { + List entries = new ArrayList<>(); + List prefixes = new ArrayList<>(); + if (maxKeys == 0) return new ListPage(entries, prefixes, null, false); + String lastKey = null, activePrefix = null; + boolean truncated = false; + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try (PreparedStatement query = connection.prepareStatement( + "SELECT object_key, length, modified, etag, sha256, content_type FROM cluster_objects WHERE bucket=? AND object_key>=? ORDER BY object_key")) { + query.setString(1, bucket); + query.setString(2, after != null && after.compareTo(prefix) > 0 ? after : prefix); + query.setFetchSize(128); + try (ResultSet result = query.executeQuery()) { + while (result.next()) { + String key = result.getString(1); + if (!key.startsWith(prefix)) break; + if (after != null && key.compareTo(after) <= 0) continue; + String group = null; + if (!delimiter.isEmpty()) { + int at = key.indexOf(delimiter, prefix.length()); + if (at >= 0) group = key.substring(0, at + delimiter.length()); + } + if (group != null && group.equals(activePrefix)) { lastKey = key; continue; } + if (entries.size() + prefixes.size() >= maxKeys) { truncated = true; break; } + if (group != null) { prefixes.add(group); activePrefix = group; } + else { + entries.add(new ListedObject(key, new Metadata(result.getLong(2), result.getLong(3), + result.getString(4), result.getBytes(5), bucket, key, result.getString(6)))); + activePrefix = null; + } + lastKey = key; + } + } + } + connection.commit(); + } catch (SQLException error) { throw databaseError(error); } + return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated); + } + + private long lockUsage(Connection connection, String bucket) throws SQLException { + try (PreparedStatement query = connection.prepareStatement("SELECT used_bytes FROM cluster_usage WHERE bucket=? FOR UPDATE")) { + query.setString(1, bucket); + try (ResultSet result = query.executeQuery()) { + if (!result.next()) throw new SQLException("Bucket quota row is missing"); + return result.getLong(1); + } + } + } + private long currentLength(Connection connection, String bucket, String key) throws SQLException { + try (PreparedStatement query = connection.prepareStatement("SELECT length FROM cluster_objects WHERE bucket=? AND object_key=?")) { + query.setString(1, bucket); query.setString(2, key); + try (ResultSet result = query.executeQuery()) { return result.next() ? result.getLong(1) : -1; } + } + } + private static void bindObject(PreparedStatement update, Metadata data, UUID generation) throws SQLException { + update.setString(1, data.bucket()); update.setString(2, data.key()); update.setObject(3, generation); + update.setLong(4, data.length()); update.setLong(5, data.modified()); update.setString(6, data.etag()); + update.setBytes(7, data.sha256()); update.setString(8, data.contentType()); + } + private static List replicaIds(ResultSet result, int column) throws SQLException, IOException { + java.sql.Array value = result.getArray(column); + if (value == null) throw new IOException("Segment has no migrated replica identities"); + try { + Object[] ids = (Object[]) value.getArray(); + List replicas = new ArrayList<>(ids.length); + for (Object id : ids) replicas.add((UUID) id); + return List.copyOf(replicas); + } finally { value.free(); } + } + private static IOException databaseError(SQLException error) { return new IOException("Metadata database operation failed", error); } + private static MessageDigest digest(String algorithm) { + try { return MessageDigest.getInstance(algorithm); } + catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); } + } + private InputStream verifiedObject(List segments, Metadata metadata) throws IOException { + Path staged = Files.createTempFile("objectstore-read-", ".pending"); + boolean ready = false; + try { + MessageDigest hash = digest("SHA-256"); + long count; + try (InputStream source = new DigestInputStream(new SegmentStream(segments), hash); + OutputStream output = Files.newOutputStream(staged)) { + count = source.transferTo(output); + } + if (count != metadata.length() || !MessageDigest.isEqual(hash.digest(), metadata.sha256())) + throw new IOException("Object manifest failed integrity verification"); + InputStream file = Files.newInputStream(staged); + ready = true; + return new FilterInputStream(file) { + @Override public void close() throws IOException { + try { super.close(); } + finally { Files.deleteIfExists(staged); } + } + }; + } finally { if (!ready) Files.deleteIfExists(staged); } + } + @Override public boolean ready() { + if (!nodes.availableHostsAtLeast(2, testNodeDomains)) return false; + try (Connection connection = connect(); var statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT 1")) { + return result.next() && result.getInt(1) == 1; + } catch (SQLException error) { return false; } + } + RepairReport repairOnce() throws IOException { + int scanned = 0, restored = 0, underReplicated = 0, unrecoverable = 0; + try (Connection reader = connect()) { + reader.setAutoCommit(false); + try (PreparedStatement query = reader.prepareStatement( + "SELECT s.generation, s.ordinal, s.segment_id, s.length, s.sha256, s.replica_ids, s.placement_version FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation ORDER BY s.generation, s.ordinal")) { + query.setFetchSize(128); + try (ResultSet result = query.executeQuery()) { + while (result.next()) { + scanned++; + RepairTarget target = new RepairTarget((UUID) result.getObject(1), result.getInt(2), + result.getLong(7), new Segment((UUID) result.getObject(3), result.getInt(4), + result.getBytes(5), replicaIds(result, 6))); + Segment segment = target.segment(); + byte[] copy = null; + Set healthy = new HashSet<>(); + Set healthyHosts = new HashSet<>(); + for (UUID id : segment.replicas()) { + int node = nodes.index(id); + if (node < 0) continue; + try { + byte[] candidate = nodes.get(node, segment.id(), segment.length(), segment.hash()); + if (copy == null) copy = candidate; + healthy.add(id); + healthyHosts.add(nodes.faultDomain(node, testNodeDomains)); + } catch (IOException error) { } + } + if (copy == null) { unrecoverable++; continue; } + for (int node : PlacementPolicy.candidates(segment.id(), nodes, testNodeDomains)) { + UUID host = nodes.faultDomain(node, testNodeDomains); + if (healthyHosts.contains(host)) continue; + try { + nodes.repair(node, segment.id(), copy, segment.hash()); + healthy.add(nodes.node(node).id()); + healthyHosts.add(host); + restored++; + } catch (IOException error) { } + if (healthyHosts.size() == 3) break; + } + if (healthyHosts.size() < 3) underReplicated++; + Set listed = new java.util.LinkedHashSet<>(segment.replicas()); + listed.addAll(healthy); + if (listed.size() != segment.replicas().size()) { + try (Connection writer = connect(); PreparedStatement update = writer.prepareStatement( + "UPDATE cluster_segments SET replica_ids=?, placement_version=placement_version+1 WHERE generation=? AND ordinal=? AND placement_version=?")) { + update.setArray(1, writer.createArrayOf("uuid", listed.toArray())); + update.setObject(2, target.generation()); + update.setInt(3, target.ordinal()); + update.setLong(4, target.version()); + update.executeUpdate(); + } + } + } + } + } + reader.commit(); + } catch (SQLException error) { throw databaseError(error); } + return new RepairReport(scanned, restored, underReplicated, unrecoverable); + } + @Override public void close() {} + + private final class SegmentStream extends InputStream { + private final List segments; + private int position; + private ByteArrayInputStream current; + private boolean closed; + SegmentStream(List segments) { this.segments = segments; } + @Override public int read() throws IOException { + byte[] one = new byte[1]; + int count = read(one, 0, 1); + return count < 0 ? -1 : one[0] & 255; + } + @Override public int read(byte[] buffer, int offset, int length) throws IOException { + if (closed) throw new IOException("Object stream is closed"); + if (length == 0) return 0; + while (current == null || current.available() == 0) { + if (position == segments.size()) return -1; + Segment segment = segments.get(position++); + IOException failure = null; + for (UUID replica : segment.replicas()) { + int node = nodes.index(replica); + if (node < 0) continue; + try { + byte[] bytes = nodes.get(node, segment.id(), segment.length(), segment.hash()); + current = new ByteArrayInputStream(bytes); + break; + } catch (IOException error) { failure = error; } + } + if (current == null || current.available() == 0) + throw new StoreException(503, "SlowDown", "No verified replica is currently available", failure); + } + return current.read(buffer, offset, length); + } + @Override public void close() { closed = true; current = null; } + } +} diff --git a/src/cloud/lunarsky/store/DiskStore.java b/src/cloud/lunarsky/store/DiskStore.java index ed2f392..fd7d7f3 100644 --- a/src/cloud/lunarsky/store/DiskStore.java +++ b/src/cloud/lunarsky/store/DiskStore.java @@ -5,28 +5,35 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.security.MessageDigest; import java.time.Instant; -import java.util.Arrays; -import java.util.HexFormat; +import java.util.*; +import cloud.lunarsky.store.ObjectStorage.Metadata; +import cloud.lunarsky.store.ObjectStorage.OpenObject; +import cloud.lunarsky.store.ObjectStorage.ListedObject; +import cloud.lunarsky.store.ObjectStorage.ListPage; -final class DiskStore implements AutoCloseable { - private static final long MAGIC = 0x4c534f424a303031L; - private static final int HEADER = 72; - private final Path objects, temporary; +final class DiskStore implements ObjectStorage { + private static final long MAGIC_V1 = 0x4c534f424a303031L; + private static final long MAGIC_V2 = 0x4c534f424a303032L; + private static final int HEADER_V1 = 72; + private static final int HEADER_V2 = 78; + private final Path root, objects, temporary; private final FileChannel lockChannel; private final FileLock processLock; private final long maxObject, maxTotal; private final Object[] locks = new Object[128]; + private final NavigableMap index = new TreeMap<>(); private long used; - record Metadata(long length, long modified, String etag, byte[] sha256) {} - record OpenObject(Metadata metadata, InputStream stream) implements AutoCloseable { - public void close() throws IOException { stream.close(); } - } + private long objectCount, legacyCount; + + record Record(Metadata metadata, int headerLength) {} DiskStore(Path root, long maxObject, long maxTotal) throws IOException { + this.root = root; objects = root.resolve("objects"); temporary = root.resolve("pending"); this.maxObject = maxObject; this.maxTotal = maxTotal; Arrays.setAll(locks, i -> new Object()); @@ -39,16 +46,25 @@ final class DiskStore implements AutoCloseable { catch (OverlappingFileLockException e) { throw new IOException("Data directory is already in use", e); } if (acquired == null) throw new IOException("Data directory is already in use"); Files.createDirectories(objects); Files.createDirectories(temporary); - try (var paths=Files.list(temporary)) { - for(Path p:paths.toList()) if(p.getFileName().toString().endsWith(".part"))Files.delete(p); + syncDirectory(root); + try (var paths = Files.list(temporary)) { + for (Path p : paths.toList()) if (p.getFileName().toString().endsWith(".part")) Files.delete(p); } - try (var paths=Files.walk(objects)) { - for(Path p:paths.filter(Files::isRegularFile).toList()) { - try(var in=new DataInputStream(Files.newInputStream(p))) { - long length = metadata(in).length(); - if(Files.size(p)-HEADER != length) throw new IOException("Truncated or oversized object record: "+p); - used = Math.addExact(used, length); - } + try (var paths = Files.walk(objects)) { + for (Path p : paths.filter(Files::isRegularFile).toList()) { + Record record; + try (var in = new DataInputStream(Files.newInputStream(p))) { record = readRecord(in); } + Metadata meta = record.metadata(); + if (Files.size(p) - record.headerLength() != meta.length()) + throw new IOException("Truncated or oversized object record: " + p); + if (meta.key() != null) { + if (!p.equals(objectPath(meta.bucket(), meta.key()))) + throw new IOException("Mismatched object record: " + p); + if (index.put(indexKey(meta.bucket(), meta.key()), meta) != null) + throw new IOException("Duplicate object record: " + p); + } else legacyCount++; + objectCount++; + used = Math.addExact(used, meta.length()); } } ready = true; @@ -66,72 +82,189 @@ final class DiskStore implements AutoCloseable { processLock.release(); lockChannel.close(); } + Path root() { return root; } + long maxObject() { return maxObject; } + long maxTotal() { return maxTotal; } + synchronized long usedBytes() { return used; } + synchronized int indexedObjects() { return index.size(); } + synchronized long objectCount() { return objectCount; } + synchronized long legacyObjects() { return legacyCount; } - private Path object(String bucket, String key) throws IOException { - String id=SigV4.hex(SigV4.hash((bucket+"/"+key).getBytes(StandardCharsets.UTF_8))); - Path shard=objects.resolve(id.substring(0,2));Files.createDirectories(shard); - return shard.resolve(id); + private static String indexKey(String bucket, String key) { return bucket + "\0" + key; } + private Path objectPath(String bucket, String key) { + String id = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(StandardCharsets.UTF_8))); + return objects.resolve(id.substring(0, 2)).resolve(id); } - private Object lock(Path p){return locks[(p.hashCode()&0x7fffffff)%locks.length];} + private synchronized Path object(String bucket, String key) throws IOException { + Path path = objectPath(bucket, key); + if (!Files.isDirectory(path.getParent())) { + Files.createDirectories(path.getParent()); + syncDirectory(objects); + } + return path; + } + static void syncDirectory(Path directory) throws IOException { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } + private Object lock(Path p) { return locks[(p.hashCode() & 0x7fffffff) % locks.length]; } - Metadata put(String bucket,String key,InputStream input,long length,String expectedHash,String checksum,boolean createOnly) throws IOException { - if(length<0)throw new StoreException(411,"MissingContentLength","Content-Length is required"); - if(length>maxObject)throw new StoreException(413,"EntityTooLarge","Object exceeds the configured size limit"); - Path destination=object(bucket,key),pending=Files.createTempFile(temporary,"upload-",".part"); + public Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, + String checksum, boolean createOnly, String contentType) throws IOException { + if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required"); + if (length > maxObject) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit"); + byte[] bucketBytes = bucket.getBytes(StandardCharsets.UTF_8); + byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); + byte[] typeBytes = contentType.getBytes(StandardCharsets.UTF_8); + if (bucketBytes.length > 63 || keyBytes.length > 1024 || typeBytes.length > 255) + throw new StoreException(400, "InvalidArgument", "Object metadata is too long"); + int headerLength = HEADER_V2 + bucketBytes.length + keyBytes.length + typeBytes.length; + Path destination = object(bucket, key), pending = Files.createTempFile(temporary, "upload-", ".part"); try { - MessageDigest sha=digest("SHA-256"),md5=digest("MD5"); - long count=0; - try(OutputStream out=Files.newOutputStream(pending)){ - out.write(new byte[HEADER]);byte[] buffer=new byte[65536];int n; - while((n=input.read(buffer))!=-1){count+=n;if(count>length||count>maxObject)throw new StoreException(413,"EntityTooLarge","Payload exceeds declared size");sha.update(buffer,0,n);md5.update(buffer,0,n);out.write(buffer,0,n);} - } - if(count!=length)throw new StoreException(400,"IncompleteBody","Payload length does not match Content-Length"); - byte[] hash=sha.digest(),etag=md5.digest(); - if(!MessageDigest.isEqual(hash,HexFormat.of().parseHex(expectedHash)))throw new StoreException(400,"XAmzContentSHA256Mismatch","Payload hash mismatch"); - if(checksum!=null&&!java.util.Base64.getEncoder().encodeToString(hash).equals(checksum))throw new StoreException(400,"BadDigest","SHA-256 checksum mismatch"); - long modified=Instant.now().toEpochMilli(); - try(FileChannel file=FileChannel.open(pending,StandardOpenOption.WRITE)){ - ByteBuffer header=ByteBuffer.allocate(HEADER).putLong(MAGIC).putLong(count).putLong(modified).put(etag).put(hash);header.flip(); - while(header.hasRemaining())file.write(header);file.force(true); - } - synchronized(lock(destination)){ - long previous=0; - if(Files.exists(destination)){ - if(createOnly)throw new StoreException(412,"PreconditionFailed","Object already exists"); - try(var in=new DataInputStream(Files.newInputStream(destination))){previous=metadata(in).length();} - } - synchronized(this){ - if(used-previous+count>maxTotal)throw new StoreException(507,"InsufficientStorage","Store capacity limit reached"); - Files.move(pending,destination,StandardCopyOption.ATOMIC_MOVE,StandardCopyOption.REPLACE_EXISTING); - used=used-previous+count; + MessageDigest sha = digest("SHA-256"), md5 = digest("MD5"); + long count = 0; + try (OutputStream out = Files.newOutputStream(pending)) { + out.write(new byte[headerLength]); + byte[] buffer = new byte[65536]; int n; + while ((n = input.read(buffer)) != -1) { + count += n; + if (count > length || count > maxObject) + throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size"); + sha.update(buffer, 0, n); md5.update(buffer, 0, n); out.write(buffer, 0, n); } } - return new Metadata(count,modified,SigV4.hex(etag),hash); - } finally {Files.deleteIfExists(pending);} + if (count != length) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length"); + byte[] hash = sha.digest(), etag = md5.digest(); + if (!MessageDigest.isEqual(hash, HexFormat.of().parseHex(expectedHash))) + throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch"); + if (checksum != null && !Base64.getEncoder().encodeToString(hash).equals(checksum)) + throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch"); + long modified = Instant.now().toEpochMilli(); + ByteBuffer header = ByteBuffer.allocate(headerLength).putLong(MAGIC_V2).putLong(count) + .putLong(modified).put(etag).put(hash).putShort((short) bucketBytes.length) + .putShort((short) keyBytes.length).putShort((short) typeBytes.length) + .put(bucketBytes).put(keyBytes).put(typeBytes); + header.flip(); + try (FileChannel file = FileChannel.open(pending, StandardOpenOption.WRITE)) { + while (header.hasRemaining()) file.write(header, header.position()); + file.force(true); + } + Metadata metadata = new Metadata(count, modified, SigV4.hex(etag), hash, bucket, key, contentType); + synchronized (lock(destination)) { + long previous = 0; + boolean existed = Files.exists(destination); + boolean legacy = false; + if (existed) { + if (createOnly) throw new StoreException(412, "PreconditionFailed", "Object already exists"); + try (var in = new DataInputStream(Files.newInputStream(destination))) { + Metadata old = readRecord(in).metadata(); + previous = old.length(); + legacy = old.key() == null; + } + } + synchronized (this) { + if (used - previous + count > maxTotal) + throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); + Files.move(pending, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + used = used - previous + count; + if (!existed) objectCount++; + if (legacy) legacyCount--; + index.put(indexKey(bucket, key), metadata); + syncDirectory(destination.getParent()); + } + } + return metadata; + } finally { Files.deleteIfExists(pending); } } - OpenObject open(String bucket,String key) throws IOException { - Path destination=object(bucket,key); - synchronized(lock(destination)){ + public OpenObject open(String bucket, String key) throws IOException { + Path destination = object(bucket, key); + synchronized (lock(destination)) { final DataInputStream input; - try{input=new DataInputStream(Files.newInputStream(destination));} - catch(NoSuchFileException e){throw new StoreException(404,"NoSuchKey","Object not found");} - try{return new OpenObject(metadata(input),input);}catch(IOException e){input.close();throw e;} + try { input = new DataInputStream(Files.newInputStream(destination)); } + catch (NoSuchFileException e) { throw new StoreException(404, "NoSuchKey", "Object not found"); } + try { return new OpenObject(readRecord(input).metadata(), input); } + catch (IOException e) { input.close(); throw e; } } } - void delete(String bucket,String key) throws IOException { - Path destination=object(bucket,key); - synchronized(lock(destination)){ - if(!Files.exists(destination))return; - long length;try(var input=new DataInputStream(Files.newInputStream(destination))){length=metadata(input).length();} - synchronized(this){Files.delete(destination);used-=length;} + + public void delete(String bucket, String key) throws IOException { + Path destination = object(bucket, key); + synchronized (lock(destination)) { + if (!Files.exists(destination)) return; + long length; + boolean legacy; + try (var input = new DataInputStream(Files.newInputStream(destination))) { + Metadata old = readRecord(input).metadata(); + length = old.length(); + legacy = old.key() == null; + } + synchronized (this) { + Files.delete(destination); + used -= length; + objectCount--; + if (legacy) legacyCount--; + index.remove(indexKey(bucket, key)); + syncDirectory(destination.getParent()); + } } } - private static Metadata metadata(DataInputStream in) throws IOException { - if(in.readLong()!=MAGIC)throw new IOException("Invalid object record"); - long length=in.readLong(),modified=in.readLong();byte[] md5=new byte[16],sha=new byte[32];in.readFully(md5);in.readFully(sha); - if(length<0)throw new IOException("Invalid object length"); - return new Metadata(length,modified,SigV4.hex(md5),sha); + + public synchronized ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) { + List entries = new ArrayList<>(); + List prefixes = new ArrayList<>(); + if (maxKeys == 0) return new ListPage(entries, prefixes, null, false); + String lastKey = null; + boolean truncated = false; + String activePrefix = null; + for (Metadata meta : index.values()) { + if (!meta.bucket().equals(bucket) || !meta.key().startsWith(prefix)) continue; + String key = meta.key(); + if (after != null && key.compareTo(after) <= 0) continue; + String group = null; + if (!delimiter.isEmpty()) { + int at = key.indexOf(delimiter, prefix.length()); + if (at >= 0) group = key.substring(0, at + delimiter.length()); + } + if (group != null && group.equals(activePrefix)) { lastKey = key; continue; } + if (entries.size() + prefixes.size() >= maxKeys) { truncated = true; break; } + if (group != null) { prefixes.add(group); activePrefix = group; } + else { entries.add(new ListedObject(key, meta)); activePrefix = null; } + lastKey = key; + } + return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated); + } + + static Record readRecord(DataInputStream in) throws IOException { + long magic = in.readLong(); + if (magic != MAGIC_V1 && magic != MAGIC_V2) throw new IOException("Invalid object record"); + long length = in.readLong(), modified = in.readLong(); + byte[] md5 = new byte[16], sha = new byte[32]; + in.readFully(md5); in.readFully(sha); + if (length < 0) throw new IOException("Invalid object record length"); + if (magic == MAGIC_V1) + return new Record(new Metadata(length, modified, SigV4.hex(md5), sha, + null, null, "application/octet-stream"), HEADER_V1); + int bucketLength = in.readUnsignedShort(), keyLength = in.readUnsignedShort(), typeLength = in.readUnsignedShort(); + if (bucketLength < 1 || bucketLength > 63 || keyLength < 1 || keyLength > 1024 || typeLength < 1 || typeLength > 255) + throw new IOException("Invalid object record metadata"); + String bucket = utf8(in.readNBytes(bucketLength)); + String key = utf8(in.readNBytes(keyLength)); + String contentType = utf8(in.readNBytes(typeLength)); + if (bucket.getBytes(StandardCharsets.UTF_8).length != bucketLength || + key.getBytes(StandardCharsets.UTF_8).length != keyLength || + contentType.getBytes(StandardCharsets.UTF_8).length != typeLength) + throw new IOException("Invalid object record metadata"); + return new Record(new Metadata(length, modified, SigV4.hex(md5), sha, + bucket, key, contentType), HEADER_V2 + bucketLength + keyLength + typeLength); + } + private static String utf8(byte[] bytes) throws IOException { + return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } + private static MessageDigest digest(String algorithm) { + try { return MessageDigest.getInstance(algorithm); } + catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException(e); } } - private static MessageDigest digest(String algorithm){try{return MessageDigest.getInstance(algorithm);}catch(java.security.NoSuchAlgorithmException e){throw new IllegalStateException(e);}} } diff --git a/src/cloud/lunarsky/store/Main.java b/src/cloud/lunarsky/store/Main.java index 0eb1281..38e3f00 100644 --- a/src/cloud/lunarsky/store/Main.java +++ b/src/cloud/lunarsky/store/Main.java @@ -3,112 +3,498 @@ package cloud.lunarsky.store; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; import java.io.IOException; +import java.io.ByteArrayInputStream; import java.net.InetSocketAddress; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.Clock; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.Base64; +import java.util.HashMap; import java.util.Map; import java.util.UUID; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; public final class Main { - private final DiskStore store; + private final ObjectStorage store; private final SigV4 authentication; private final String bucket; - private final Semaphore slots=new Semaphore(16); - Main(DiskStore store,SigV4 authentication,String bucket){this.store=store;this.authentication=authentication;this.bucket=bucket;} + private final MultipartStorage multipart; + private final Semaphore slots = new Semaphore(16); + + Main(DiskStore store, SigV4 authentication, String bucket) throws IOException { + this(store, new MultipartStore(store), authentication, bucket); + } + + Main(ObjectStorage store, MultipartStorage multipart, SigV4 authentication, String bucket) { + this.store = store; this.multipart = multipart; + this.authentication = authentication; this.bucket = bucket; + } void handle(HttpExchange exchange) throws IOException { - boolean admitted=slots.tryAcquire(); - String requestId=UUID.randomUUID().toString(); - exchange.getResponseHeaders().set("x-amz-request-id",requestId); - exchange.getResponseHeaders().set("X-Content-Type-Options","nosniff"); + boolean admitted = slots.tryAcquire(); + String requestId = UUID.randomUUID().toString(); + exchange.getResponseHeaders().set("x-amz-request-id", requestId); + exchange.getResponseHeaders().set("X-Content-Type-Options", "nosniff"); try { - if(!admitted)throw new StoreException(503,"SlowDown","Too many concurrent requests"); - if(exchange.getRequestURI().getRawPath().equals("/health")&&exchange.getRequestMethod().equals("GET")){ - byte[] body="{\"status\":\"ok\",\"service\":\"lunarsky-objectstore\"}".getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().set("Content-Type","application/json");exchange.sendResponseHeaders(200,body.length);exchange.getResponseBody().write(body);return; + if (!admitted) throw new StoreException(503, "SlowDown", "Too many concurrent requests"); + if (exchange.getRequestURI().getRawPath().equals("/health") && exchange.getRequestMethod().equals("GET")) { + byte[] body = "{\"status\":\"ok\",\"service\":\"lunarsky-objectstore\"}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + return; } - String hash=authentication.verify(exchange.getRequestMethod(),exchange.getRequestURI(),exchange.getRequestHeaders()); - String path=SigV4.decode(exchange.getRequestURI().getRawPath()); - String prefix="/"+bucket+"/"; - if(!path.startsWith(prefix))throw new StoreException(404,"NoSuchBucket","Bucket not found"); - String key=path.substring(prefix.length()); - if(key.isEmpty()||key.getBytes(StandardCharsets.UTF_8).length>1024||key.indexOf('\0')>=0)throw new StoreException(400,"InvalidArgument","Invalid object key"); - String query=exchange.getRequestURI().getRawQuery(); - if(query!=null&&!query.isEmpty()&&!query.matches("x-id=(PutObject|GetObject|HeadObject|DeleteObject)"))unsupported("Query operation"); - var headers=exchange.getRequestHeaders(); - if(headers.containsKey("range")||headers.containsKey("if-match")||headers.containsKey("if-modified-since")||headers.containsKey("if-unmodified-since"))unsupported("Range or conditional read"); - for(String name:headers.keySet()){ - String lower=name.toLowerCase(java.util.Locale.ROOT); - if(lower.startsWith("x-amz-")&&!java.util.Set.of("x-amz-date","x-amz-content-sha256","x-amz-checksum-sha256","x-amz-sdk-checksum-algorithm","x-amz-user-agent").contains(lower))unsupported("Amazon header"); - if(lower.startsWith("x-amz-meta-")||lower.startsWith("x-amz-server-side-")||lower.startsWith("x-amz-copy-")||lower.startsWith("x-amz-acl")||lower.startsWith("x-amz-grant")||lower.startsWith("x-amz-tagging")||lower.equals("content-md5"))unsupported("Object metadata, encryption, ACL, copy, tagging or MD5 header"); - if(lower.startsWith("x-amz-checksum-")&&!lower.equals("x-amz-checksum-sha256"))unsupported("Checksum algorithm"); + if (exchange.getRequestURI().getRawPath().equals("/ready") && exchange.getRequestMethod().equals("GET")) { + boolean ready = store.ready(); + byte[] body = (ready ? "ready" : "unavailable").getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "text/plain; charset=utf-8"); + exchange.sendResponseHeaders(ready ? 200 : 503, body.length); + exchange.getResponseBody().write(body); + return; } - String method=exchange.getRequestMethod(); - String algorithm=SigV4.single(headers,"x-amz-sdk-checksum-algorithm"); - if(algorithm!=null&&!algorithm.equals("SHA256"))unsupported("Checksum algorithm"); - if(!method.equals("PUT")&&(headers.containsKey("transfer-encoding")||(headers.containsKey("content-length")&&!"0".equals(SigV4.single(headers,"content-length")))))throw new StoreException(400,"InvalidRequest","Read/delete requests must have empty bodies"); - if(!method.equals("PUT")&&!hash.equals(SigV4.hex(SigV4.hash(new byte[0]))))throw new StoreException(400,"InvalidRequest","Read/delete requests must have empty bodies"); - switch(method){ + String hash = authentication.verify(exchange.getRequestMethod(), exchange.getRequestURI(), exchange.getRequestHeaders()); + String path = SigV4.decode(exchange.getRequestURI().getRawPath()); + Map query = query(exchange.getRequestURI().getRawQuery()); + if (path.equals("/" + bucket) || path.equals("/" + bucket + "/")) { + if (!exchange.getRequestMethod().equals("GET") || !"2".equals(query.get("list-type")) || + !query.keySet().stream().allMatch(java.util.Set.of("list-type", "prefix", "delimiter", "max-keys", + "continuation-token", "start-after", "encoding-type", "x-id")::contains) || + (query.containsKey("x-id") && !"ListObjectsV2".equals(query.get("x-id")))) + unsupported("Bucket operation"); + requireEmptyBody(exchange, hash); + listObjects(exchange, query); + return; + } + String prefix = "/" + bucket + "/"; + if (!path.startsWith(prefix)) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + String key = path.substring(prefix.length()); + if (key.isEmpty() || key.getBytes(StandardCharsets.UTF_8).length > 1024 || key.indexOf('\0') >= 0) + throw new StoreException(400, "InvalidArgument", "Invalid object key"); + String method = exchange.getRequestMethod(); + boolean multipartRequest = multipartRequest(method, query); + if (!multipartRequest && !query.isEmpty() && !(query.size() == 1 && + ("PutObject".equals(query.get("x-id")) || "GetObject".equals(query.get("x-id")) || + "HeadObject".equals(query.get("x-id")) || "DeleteObject".equals(query.get("x-id"))))) + unsupported("Query operation"); + var headers = exchange.getRequestHeaders(); + for (String name : headers.keySet()) { + String lower = name.toLowerCase(java.util.Locale.ROOT); + if (lower.startsWith("x-amz-") && !java.util.Set.of("x-amz-date", "x-amz-content-sha256", + "x-amz-checksum-sha256", "x-amz-sdk-checksum-algorithm", "x-amz-user-agent").contains(lower)) + unsupported("Amazon header"); + if (lower.startsWith("x-amz-meta-") || lower.startsWith("x-amz-server-side-") || + lower.startsWith("x-amz-copy-") || lower.startsWith("x-amz-acl") || + lower.startsWith("x-amz-grant") || lower.startsWith("x-amz-tagging") || + lower.equals("content-md5")) unsupported("Object metadata, encryption, ACL, copy, tagging or MD5 header"); + if (lower.startsWith("x-amz-checksum-") && !lower.equals("x-amz-checksum-sha256")) + unsupported("Checksum algorithm"); + } + String algorithm = SigV4.single(headers, "x-amz-sdk-checksum-algorithm"); + if (algorithm != null && !algorithm.equals("SHA256")) unsupported("Checksum algorithm"); + if (multipartRequest) { + handleMultipart(exchange, method, query, key, hash); + return; + } + if (!method.equals("PUT")) requireEmptyBody(exchange, hash); + switch (method) { case "PUT" -> { - String length=SigV4.single(headers,"content-length"),condition=SigV4.single(headers,"if-none-match"); - if(condition!=null&&!condition.equals("*"))unsupported("Write condition"); - long bytes;try{bytes=length==null?-1:Long.parseLong(length);}catch(NumberFormatException e){throw new StoreException(400,"InvalidArgument","Invalid Content-Length");} - if(headers.containsKey("content-encoding"))unsupported("Encoded payload"); - DiskStore.Metadata data=store.put(bucket,key,exchange.getRequestBody(),bytes,hash,SigV4.single(headers,"x-amz-checksum-sha256"),condition!=null); - exchange.getResponseHeaders().set("ETag","\""+data.etag()+"\""); - exchange.getResponseHeaders().set("x-amz-checksum-sha256",java.util.Base64.getEncoder().encodeToString(data.sha256())); - exchange.sendResponseHeaders(200,-1); + String length = SigV4.single(headers, "content-length"), condition = SigV4.single(headers, "if-none-match"); + if (condition != null && !condition.equals("*")) unsupported("Write condition"); + long bytes; + try { bytes = length == null ? -1 : Long.parseLong(length); } + catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid Content-Length"); } + if (headers.containsKey("content-encoding")) unsupported("Encoded payload"); + String contentType = contentType(headers); + ObjectStorage.Metadata data = store.put(bucket, key, exchange.getRequestBody(), bytes, hash, + SigV4.single(headers, "x-amz-checksum-sha256"), condition != null, contentType); + exchange.getResponseHeaders().set("ETag", "\"" + data.etag() + "\""); + exchange.getResponseHeaders().set("x-amz-checksum-sha256", Base64.getEncoder().encodeToString(data.sha256())); + exchange.sendResponseHeaders(200, -1); } - case "GET", "HEAD" -> { - if(headers.containsKey("if-none-match"))unsupported("Conditional read"); - try(var object=store.open(bucket,key)){ - var meta=object.metadata(); - exchange.getResponseHeaders().set("Content-Type","application/octet-stream"); - exchange.getResponseHeaders().set("Content-Length",Long.toString(meta.length())); - exchange.getResponseHeaders().set("ETag","\""+meta.etag()+"\""); - exchange.getResponseHeaders().set("Last-Modified",DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC).format(Instant.ofEpochMilli(meta.modified()))); - if(method.equals("HEAD")||meta.length()==0)exchange.sendResponseHeaders(200,-1); - else{exchange.sendResponseHeaders(200,meta.length());object.stream().transferTo(exchange.getResponseBody());} - } + case "GET", "HEAD" -> readObject(exchange, key); + case "DELETE" -> { + if (headers.containsKey("if-none-match")) unsupported("Conditional delete"); + store.delete(bucket, key); + exchange.sendResponseHeaders(204, -1); } - case "DELETE" -> {if(headers.containsKey("if-none-match"))unsupported("Conditional delete");store.delete(bucket,key);exchange.sendResponseHeaders(204,-1);} default -> unsupported("HTTP method"); } - } catch(StoreException error){sendError(exchange,error.status,error.code,error.getMessage(),requestId);} - catch(Exception error){System.err.println("ObjectStore request failed: "+requestId+" "+error.getClass().getSimpleName());sendError(exchange,500,"InternalError","Storage operation failed",requestId);} - finally {if(admitted)slots.release();exchange.close();} + } catch (StoreException error) { sendError(exchange, error.status, error.code, error.getMessage(), requestId); } + catch (Exception error) { + System.err.println("ObjectStore request failed: " + requestId + " " + error.getClass().getSimpleName()); + sendError(exchange, 500, "InternalError", "Storage operation failed", requestId); + } finally { if (admitted) slots.release(); exchange.close(); } } - private static void unsupported(String feature){throw new StoreException(501,"NotImplemented",feature+" is not supported in this prototype");} - private static String xml(String text){return text.replace("&","&").replace("<","<").replace(">",">").replace("\"",""");} - private static void sendError(HttpExchange exchange,int status,String code,String message,String id)throws IOException{ - if(exchange.getResponseCode()!=-1)return; - byte[] body=(""+xml(code)+""+xml(message)+""+id+"").getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().set("Content-Type","application/xml");exchange.sendResponseHeaders(status,exchange.getRequestMethod().equals("HEAD")?-1:body.length); - if(!exchange.getRequestMethod().equals("HEAD"))exchange.getResponseBody().write(body); + + private static String contentType(com.sun.net.httpserver.Headers headers) { + String value = SigV4.single(headers, "content-type"); + if (value == null) return "application/octet-stream"; + if (value.isBlank() || value.getBytes(StandardCharsets.UTF_8).length > 255 || + !value.chars().allMatch(c -> c >= 32 && c <= 126)) + throw new StoreException(400, "InvalidArgument", "Invalid Content-Type"); + return value; } - public static void main(String[] args)throws Exception{ - Map env=System.getenv(); - String access=required(env,"S3_ACCESS_KEY"),secret=required(env,"S3_SECRET_KEY"),bucket=env.getOrDefault("S3_BUCKET","lunaris-files"),region=env.getOrDefault("S3_REGION","us-east-1"); - if(!access.matches("[A-Za-z0-9]{16,128}")||secret.length()<32||!bucket.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]"))throw new IllegalArgumentException("Invalid storage credentials/bucket configuration"); - long maxObject=Long.parseLong(env.getOrDefault("MAX_OBJECT_BYTES","10485760")),maxTotal=Long.parseLong(env.getOrDefault("MAX_TOTAL_BYTES","2147483648")); - if(maxObject<1||maxObject>1073741824L||maxTotal{ + + private static void requireEmptyBody(HttpExchange exchange, String hash) { + var headers = exchange.getRequestHeaders(); + if (headers.containsKey("transfer-encoding") || + (headers.containsKey("content-length") && !"0".equals(SigV4.single(headers, "content-length"))) || + !hash.equals(SigV4.hex(SigV4.hash(new byte[0])))) + throw new StoreException(400, "InvalidRequest", "Request must have an empty body"); + } + + private static boolean multipartRequest(String method, Map query) { + if (query.containsKey("uploads")) + return method.equals("POST") && query.get("uploads").isEmpty() && + query.keySet().stream().allMatch(java.util.Set.of("uploads", "x-id")::contains) && + (!query.containsKey("x-id") || query.get("x-id").equals("CreateMultipartUpload")); + if (!query.containsKey("uploadId") || + !query.keySet().stream().allMatch(java.util.Set.of("uploadId", "partNumber", "x-id")::contains)) + return false; + String xId = query.get("x-id"); + if (method.equals("PUT")) return query.containsKey("partNumber") && + (xId == null || xId.equals("UploadPart")); + if (query.containsKey("partNumber")) return false; + return (method.equals("POST") && (xId == null || xId.equals("CompleteMultipartUpload"))) || + (method.equals("DELETE") && (xId == null || xId.equals("AbortMultipartUpload"))); + } + + private void handleMultipart(HttpExchange exchange, String method, Map query, + String key, String hash) throws IOException { + var headers = exchange.getRequestHeaders(); + if (headers.containsKey("content-encoding") || headers.containsKey("if-none-match")) + unsupported("Multipart request header"); + if (query.containsKey("uploads")) { + requireEmptyBody(exchange, hash); + String id = multipart.create(bucket, key, contentType(headers)); + sendXml(exchange, 200, "" + xml(bucket) + + "" + xml(key) + "" + id + + ""); + return; + } + String id = query.get("uploadId"); + switch (method) { + case "PUT" -> { + int number; + try { number = Integer.parseInt(query.get("partNumber")); } + catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid part number"); } + long length = contentLength(headers); + String etag = multipart.putPart(id, bucket, key, number, exchange.getRequestBody(), length, + hash, SigV4.single(headers, "x-amz-checksum-sha256")); + exchange.getResponseHeaders().set("ETag", "\"" + etag + "\""); + exchange.sendResponseHeaders(200, -1); + } + case "POST" -> { + byte[] body = signedBody(exchange, hash, 65536); + List parts = completedParts(body); + var meta = multipart.complete(id, bucket, key, parts); + sendXml(exchange, 200, "" + xml(bucket) + + "" + xml(key) + """ + meta.etag() + + """); + } + case "DELETE" -> { + requireEmptyBody(exchange, hash); + multipart.abort(id, bucket, key); + exchange.sendResponseHeaders(204, -1); + } + default -> unsupported("Multipart operation"); + } + } + + private static long contentLength(com.sun.net.httpserver.Headers headers) { + String text = SigV4.single(headers, "content-length"); + if (text == null) return -1; + try { return Long.parseLong(text); } + catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid Content-Length"); } + } + private static byte[] signedBody(HttpExchange exchange, String hash, int limit) throws IOException { + long length = contentLength(exchange.getRequestHeaders()); + if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required"); + if (length > limit) throw new StoreException(413, "EntityTooLarge", "Request body is too large"); + byte[] body = exchange.getRequestBody().readNBytes(limit + 1); + if (body.length != length) throw new StoreException(400, "IncompleteBody", "Body length does not match Content-Length"); + if (!SigV4.hex(SigV4.hash(body)).equals(hash)) + throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch"); + return body; + } + private static List completedParts(byte[] body) { + try { + var factory = javax.xml.parsers.DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setExpandEntityReferences(false); + var document = factory.newDocumentBuilder().parse(new ByteArrayInputStream(body)); + if (!document.getDocumentElement().getNodeName().equals("CompleteMultipartUpload")) + throw new IllegalArgumentException(); + var nodes = document.getDocumentElement().getChildNodes(); + List parts = new ArrayList<>(); + for (int i = 0; i < nodes.getLength(); i++) { + if (!(nodes.item(i) instanceof org.w3c.dom.Element element)) continue; + if (!element.getTagName().equals("Part")) throw new IllegalArgumentException(); + String number = null, etag = null; + var fields = element.getChildNodes(); + for (int j = 0; j < fields.getLength(); j++) { + if (!(fields.item(j) instanceof org.w3c.dom.Element field)) continue; + if (field.getTagName().equals("PartNumber")) number = field.getTextContent().trim(); + else if (field.getTagName().equals("ETag")) etag = field.getTextContent().trim(); + else throw new IllegalArgumentException(); + } + if (number == null || etag == null || !etag.matches("\"?[0-9a-f]{32}\"?")) + throw new IllegalArgumentException(); + parts.add(new MultipartStorage.Part(Integer.parseInt(number), etag)); + if (parts.size() > 10000) throw new IllegalArgumentException(); + } + return parts; + } catch (Exception error) { + throw new StoreException(400, "MalformedXML", "Invalid multipart completion body"); + } + } + private static void sendXml(HttpExchange exchange, int status, String xml) throws IOException { + byte[] body = ("" + xml).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/xml"); + exchange.sendResponseHeaders(status, body.length); + exchange.getResponseBody().write(body); + } + + private void readObject(HttpExchange exchange, String key) throws IOException { + var headers = exchange.getRequestHeaders(); + if (headers.containsKey("if-match") || headers.containsKey("if-modified-since") || + headers.containsKey("if-unmodified-since")) unsupported("Conditional read"); + try (var object = store.open(bucket, key)) { + var meta = object.metadata(); + String etag = "\"" + meta.etag() + "\""; + String noneMatch = SigV4.single(headers, "if-none-match"); + if (noneMatch != null && (noneMatch.equals("*") || + java.util.Arrays.stream(noneMatch.split(",")).map(String::trim).anyMatch(etag::equals))) { + exchange.getResponseHeaders().set("ETag", etag); + exchange.sendResponseHeaders(304, -1); + return; + } + Range range; + try { range = range(SigV4.single(headers, "range"), meta.length()); } + catch (StoreException error) { + if (error.status == 416) exchange.getResponseHeaders().set("Content-Range", "bytes */" + meta.length()); + throw error; + } + var response = exchange.getResponseHeaders(); + response.set("Content-Type", meta.contentType()); + response.set("Content-Length", Long.toString(range.length())); + response.set("Accept-Ranges", "bytes"); + response.set("ETag", etag); + response.set("Last-Modified", DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) + .format(Instant.ofEpochMilli(meta.modified()))); + if (range.partial()) response.set("Content-Range", "bytes " + range.start() + "-" + range.end() + "/" + meta.length()); + int status = range.partial() ? 206 : 200; + if (exchange.getRequestMethod().equals("HEAD") || range.length() == 0) + exchange.sendResponseHeaders(status, -1); + else { + object.stream().skipNBytes(range.start()); + exchange.sendResponseHeaders(status, range.length()); + byte[] buffer = new byte[65536]; long left = range.length(); + while (left > 0) { + int n = object.stream().read(buffer, 0, (int) Math.min(buffer.length, left)); + if (n < 0) throw new IOException("Object body ended before its recorded length"); + exchange.getResponseBody().write(buffer, 0, n); + left -= n; + } + } + } + } + + private record Range(long start, long end, boolean partial) { + long length() { return end < start ? 0 : end - start + 1; } + } + private static Range range(String header, long size) { + if (header == null) return new Range(0, size - 1, false); + if (!header.matches("bytes=[0-9]*-[0-9]*") || header.equals("bytes=-") || size == 0) + throw new StoreException(416, "InvalidRange", "The requested range is not satisfiable"); + String[] parts = header.substring(6).split("-", -1); + try { + long start, end; + if (parts[0].isEmpty()) { + long suffix = Long.parseLong(parts[1]); + if (suffix == 0) throw new NumberFormatException(); + start = Math.max(0, size - suffix); end = size - 1; + } else { + start = Long.parseLong(parts[0]); + end = parts[1].isEmpty() ? size - 1 : Math.min(Long.parseLong(parts[1]), size - 1); + } + if (start >= size || end < start) throw new NumberFormatException(); + return new Range(start, end, true); + } catch (NumberFormatException e) { + throw new StoreException(416, "InvalidRange", "The requested range is not satisfiable"); + } + } + + private void listObjects(HttpExchange exchange, Map query) throws IOException { + String prefix = query.getOrDefault("prefix", ""), delimiter = query.getOrDefault("delimiter", ""); + String encoding = query.get("encoding-type"); + if (encoding != null && !encoding.equals("url")) unsupported("Encoding type"); + if (query.containsKey("continuation-token") && query.containsKey("start-after")) + throw new StoreException(400, "InvalidArgument", "Use either continuation-token or start-after"); + int maxKeys; + try { maxKeys = Integer.parseInt(query.getOrDefault("max-keys", "1000")); } + catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid max-keys"); } + if (maxKeys < 0 || maxKeys > 1000) throw new StoreException(400, "InvalidArgument", "Invalid max-keys"); + String after = query.get("start-after"); + if (query.containsKey("continuation-token")) { + try { + byte[] decoded = Base64.getUrlDecoder().decode(query.get("continuation-token")); + after = StandardCharsets.UTF_8.newDecoder().onMalformedInput(java.nio.charset.CodingErrorAction.REPORT) + .decode(java.nio.ByteBuffer.wrap(decoded)).toString(); + } catch (IllegalArgumentException | java.nio.charset.CharacterCodingException e) { + throw new StoreException(400, "InvalidArgument", "Invalid continuation token"); + } + } + var page = store.list(bucket, prefix, delimiter, maxKeys, after); + StringBuilder xml = new StringBuilder(""); + xml.append("").append(xml(bucket)).append("").append(xml(listKey(prefix, encoding))).append(""); + if (!delimiter.isEmpty()) xml.append("").append(xml(listKey(delimiter, encoding))).append(""); + if (encoding != null) xml.append("url"); + if (query.containsKey("continuation-token")) xml.append("") + .append(xml(query.get("continuation-token"))).append(""); + if (query.containsKey("start-after")) xml.append("") + .append(xml(listKey(query.get("start-after"), encoding))).append(""); + xml.append("").append(page.keyCount()).append("").append(maxKeys) + .append("").append(page.truncated()).append(""); + int objectAt = 0, prefixAt = 0; + while (objectAt < page.objects().size() || prefixAt < page.prefixes().size()) { + if (objectAt < page.objects().size() && + (prefixAt == page.prefixes().size() || + page.objects().get(objectAt).key().compareTo(page.prefixes().get(prefixAt)) < 0)) { + var entry = page.objects().get(objectAt++); + var meta = entry.metadata(); + xml.append("").append(xml(listKey(entry.key(), encoding))).append("") + .append(Instant.ofEpochMilli(meta.modified())).append(""") + .append(meta.etag()).append(""").append(meta.length()) + .append("STANDARD"); + } else { + xml.append("") + .append(xml(listKey(page.prefixes().get(prefixAt++), encoding))) + .append(""); + } + } + if (page.truncated()) xml.append("") + .append(Base64.getUrlEncoder().withoutPadding().encodeToString(page.nextKey().getBytes(StandardCharsets.UTF_8))) + .append(""); + xml.append(""); + byte[] body = xml.toString().getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/xml"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + } + + private static String listKey(String key, String encoding) { + return encoding == null ? key : SigV4.encode(key, false); + } + private static Map query(String raw) { + Map result = new HashMap<>(); + if (raw == null || raw.isEmpty()) return result; + for (String part : raw.split("&", -1)) { + String[] pair = part.split("=", 2); + String name = SigV4.decode(pair[0]); + String value = SigV4.decode(pair.length == 2 ? pair[1] : ""); + if (result.put(name, value) != null) + throw new StoreException(400, "InvalidArgument", "Duplicate query parameter"); + } + return result; + } + private static void unsupported(String feature) { throw new StoreException(501, "NotImplemented", feature + " is not supported"); } + private static String xml(String text) { + return text.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """); + } + private static void sendError(HttpExchange exchange, int status, String code, String message, String id) throws IOException { + if (exchange.getResponseCode() != -1) return; + byte[] body = ("" + xml(code) + + "" + xml(message) + "" + id + "") + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/xml"); + exchange.sendResponseHeaders(status, exchange.getRequestMethod().equals("HEAD") ? -1 : body.length); + if (!exchange.getRequestMethod().equals("HEAD")) exchange.getResponseBody().write(body); + } + public static void main(String[] args) throws Exception { + Map env = System.getenv(); + String access = required(env, "S3_ACCESS_KEY"), secret = required(env, "S3_SECRET_KEY"); + String bucket = env.getOrDefault("S3_BUCKET", "lunaris-files"), region = env.getOrDefault("S3_REGION", "us-east-1"); + if (!access.matches("[A-Za-z0-9]{16,128}") || secret.length() < 32 || + !bucket.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]")) + throw new IllegalArgumentException("Invalid storage credentials/bucket configuration"); + long maxObject = Long.parseLong(env.getOrDefault("MAX_OBJECT_BYTES", "134217728")); + long maxTotal = Long.parseLong(env.getOrDefault("MAX_TOTAL_BYTES", "2147483648")); + if (maxObject < 1 || maxObject > 1073741824L || maxTotal < maxObject) + throw new IllegalArgumentException("Invalid size limits"); + String mode = env.getOrDefault("STORE_MODE", "disk"); + ObjectStorage store; + MultipartStorage multipart; + if (mode.equals("cluster")) { + if (!"true".equals(env.get("CLUSTER_LOCAL_DEV"))) + throw new IllegalArgumentException("Cluster mode is local development only; set CLUSTER_LOCAL_DEV=true"); + String[] urls = required(env, "CLUSTER_NODES").split(",", -1); + if (urls.length < 2) throw new IllegalArgumentException("CLUSTER_NODES requires at least two URLs"); + store = new ClusterStore(required(env, "POSTGRES_JDBC_URL"), required(env, "POSTGRES_USER"), + required(env, "POSTGRES_PASSWORD"), bucket, + java.util.Arrays.stream(urls).map(URI::create).toList(), required(env, "CLUSTER_TOKEN"), null, + maxObject, maxTotal, "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS"))); + multipart = new UnavailableMultipart(); + } else if (mode.equals("disk")) { + DiskStore disk = new DiskStore(Path.of(env.getOrDefault("DATA_DIR", "/data")), maxObject, maxTotal); + store = disk; + multipart = new MultipartStore(disk); + } else throw new IllegalArgumentException("Invalid STORE_MODE"); + var app = new Main(store, multipart, new SigV4(access, secret, region, Clock.systemUTC()), bucket); + int port = Integer.parseInt(env.getOrDefault("PORT", "9000")); + var server = HttpServer.create(mode.equals("cluster") + ? new InetSocketAddress(env.getOrDefault("BIND_ADDRESS", "127.0.0.1"), port) + : new InetSocketAddress(port), 64); + var executor = Executors.newVirtualThreadPerTaskExecutor(); + server.setExecutor(executor); server.createContext("/", app::handle); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { server.stop(5); executor.close(); try { store.close(); } - catch (IOException error) { System.err.println("Could not release ObjectStore data lock: "+error.getMessage()); } + catch (IOException error) { System.err.println("Could not release ObjectStore data lock: " + error.getMessage()); } })); - server.start();System.out.println("LunarSky ObjectStore listening; S3 object-operation prototype, bucket="+bucket); + server.start(); + if (store instanceof DiskStore disk) printStartup(disk, app.multipart, bucket, region, port); + else System.out.println("ObjectStore local cluster prototype v" + Version.VALUE + " listening on :" + port); + } + private static void printStartup(DiskStore store, MultipartStorage multipart, + String bucket, String region, int port) { + System.out.println(" *"); + System.out.println(" / \\ LUNARSKY"); + System.out.println(" / L \\ ObjectStore"); + System.out.println(" /_____\\ v" + Version.VALUE); + System.out.println(); + System.out.println(" Bucket " + bucket + " (" + region + ")"); + System.out.println(" Objects " + store.objectCount() + " total, " + + store.indexedObjects() + " indexed, " + size(store.usedBytes()) + " / " + size(store.maxTotal())); + if (store.legacyObjects() > 0) + System.out.println(" Legacy " + store.legacyObjects() + " objects without stored keys"); + System.out.println(" Multipart " + multipart.activeUploads() + " active, " + + size(multipart.stagedBytes()) + " staged"); + System.out.println(" Listening :" + port); + } + private static String size(long bytes) { + if (bytes < 1024) return bytes + " B"; + String[] units = {"KiB", "MiB", "GiB", "TiB"}; + double value = bytes; + int unit = -1; + do { value /= 1024; unit++; } while (value >= 1024 && unit < units.length - 1); + return String.format(Locale.ROOT, "%.1f %s", value, units[unit]); + } + private static String required(Map env, String key) { + String value = env.get(key); + if (value == null || value.isBlank()) throw new IllegalArgumentException("Missing " + key); + return value; } - private static String required(Map env,String key){String value=env.get(key);if(value==null||value.isBlank())throw new IllegalArgumentException("Missing "+key);return value;} } diff --git a/src/cloud/lunarsky/store/MultipartStorage.java b/src/cloud/lunarsky/store/MultipartStorage.java new file mode 100644 index 0000000..7caf0b2 --- /dev/null +++ b/src/cloud/lunarsky/store/MultipartStorage.java @@ -0,0 +1,17 @@ +package cloud.lunarsky.store; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +interface MultipartStorage { + record Part(int number, String etag) {} + + String create(String bucket, String key, String contentType) throws IOException; + String putPart(String id, String bucket, String key, int number, InputStream input, + long length, String expectedHash, String checksum) throws IOException; + ObjectStorage.Metadata complete(String id, String bucket, String key, List parts) throws IOException; + void abort(String id, String bucket, String key) throws IOException; + int activeUploads(); + long stagedBytes(); +} diff --git a/src/cloud/lunarsky/store/MultipartStore.java b/src/cloud/lunarsky/store/MultipartStore.java new file mode 100644 index 0000000..5762d67 --- /dev/null +++ b/src/cloud/lunarsky/store/MultipartStore.java @@ -0,0 +1,217 @@ +package cloud.lunarsky.store; + +import java.io.*; +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; +import cloud.lunarsky.store.MultipartStorage.Part; + +final class MultipartStore implements MultipartStorage { + private static final int MAGIC = 0x4c534d50; + private final DiskStore store; + private final Path root; + private long staged; + private int active; + + private record Upload(String bucket, String key, String contentType) {} + + MultipartStore(DiskStore store) throws IOException { + this.store = store; + root = store.root().resolve("multipart"); + Files.createDirectories(root); + DiskStore.syncDirectory(store.root()); + try (var uploads = Files.list(root)) { + for (Path dir : uploads.toList()) { + if (dir.getFileName().toString().startsWith(".creating-")) { + discardCreating(dir); + continue; + } + if (!Files.isDirectory(dir)) throw new IOException("Invalid multipart upload entry: " + dir); + readUpload(dir); + active++; + try (var files = Files.list(dir)) { + for (Path file : files.toList()) { + if (file.getFileName().toString().matches("part-[0-9]{5}")) + staged = Math.addExact(staged, Files.size(file)); + else if (!file.getFileName().toString().equals("manifest")) + throw new IOException("Invalid multipart upload entry: " + file); + } + } + } + } + if (staged > store.maxTotal()) throw new IOException("Multipart staging limit exceeded"); + } + + public synchronized String create(String bucket, String key, String contentType) throws IOException { + if (active >= 32) throw new StoreException(503, "SlowDown", "Too many active uploads"); + String id = UUID.randomUUID().toString(); + Path pending = root.resolve(".creating-" + id), dir = root.resolve(id); + Files.createDirectory(pending); + try { + try (var output = new DataOutputStream(Files.newOutputStream(pending.resolve("manifest"), StandardOpenOption.CREATE_NEW))) { + output.writeInt(MAGIC); + output.writeUTF(bucket); + output.writeUTF(key); + output.writeUTF(contentType); + } + try (var channel = java.nio.channels.FileChannel.open(pending.resolve("manifest"), StandardOpenOption.READ)) { + channel.force(true); + } + DiskStore.syncDirectory(pending); + Files.move(pending, dir, StandardCopyOption.ATOMIC_MOVE); + DiskStore.syncDirectory(root); + } catch (IOException error) { + discardCreating(pending); + discardCreating(dir); + throw error; + } + active++; + return id; + } + + private void discardCreating(Path dir) throws IOException { + if (!Files.exists(dir)) return; + if (Files.isDirectory(dir)) { + try (var files = Files.list(dir)) { + for (Path file : files.toList()) Files.delete(file); + } + } + Files.delete(dir); + DiskStore.syncDirectory(root); + } + + public synchronized int activeUploads() { return active; } + public synchronized long stagedBytes() { return staged; } + + public synchronized String putPart(String id, String bucket, String key, int number, InputStream input, + long length, String expectedHash, String checksum) throws IOException { + if (number < 1 || number > 10000) throw new StoreException(400, "InvalidArgument", "Invalid part number"); + if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required"); + if (length > store.maxObject()) throw new StoreException(413, "EntityTooLarge", "Part exceeds the object limit"); + Path dir = upload(id, bucket, key), target = part(dir, number); + long previous = Files.exists(target) ? Files.size(target) : 0; + if (staged - previous + length > store.maxTotal()) + throw new StoreException(507, "InsufficientStorage", "Multipart staging limit reached"); + Path pending = Files.createTempFile(store.root().resolve("pending"), "part-", ".part"); + try { + MessageDigest sha = digest("SHA-256"), md5 = digest("MD5"); + long count = 0; + try (var output = Files.newOutputStream(pending)) { + byte[] buffer = new byte[65536]; int n; + while ((n = input.read(buffer)) != -1) { + count += n; + if (count > length) throw new StoreException(413, "EntityTooLarge", "Part exceeds declared size"); + sha.update(buffer, 0, n); md5.update(buffer, 0, n); output.write(buffer, 0, n); + } + } + if (count != length) throw new StoreException(400, "IncompleteBody", "Part length does not match Content-Length"); + byte[] actual = sha.digest(); + if (!MessageDigest.isEqual(actual, HexFormat.of().parseHex(expectedHash))) + throw new StoreException(400, "XAmzContentSHA256Mismatch", "Part hash mismatch"); + if (checksum != null && !Base64.getEncoder().encodeToString(actual).equals(checksum)) + throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch"); + try (var channel = java.nio.channels.FileChannel.open(pending, StandardOpenOption.WRITE)) { channel.force(true); } + Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + staged = staged - previous + length; + DiskStore.syncDirectory(dir); + return SigV4.hex(md5.digest()); + } finally { Files.deleteIfExists(pending); } + } + + public synchronized ObjectStorage.Metadata complete(String id, String bucket, String key, List parts) throws IOException { + Path dir = upload(id, bucket, key); + if (parts.isEmpty() || parts.size() > 10000) + throw new StoreException(400, "InvalidPart", "No valid parts supplied"); + MessageDigest sha = digest("SHA-256"); + List paths = new ArrayList<>(); + long total = 0; int last = 0; + for (Part part : parts) { + if (part.number() <= last || part.number() > 10000) + throw new StoreException(400, "InvalidPartOrder", "Parts must be in ascending order"); + last = part.number(); + Path file = part(dir, part.number()); + if (!Files.isRegularFile(file)) throw new StoreException(400, "InvalidPart", "Missing part"); + long length = Files.size(file); + total += length; + if (total > store.maxObject()) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit"); + MessageDigest md5 = digest("MD5"); + try (var input = Files.newInputStream(file)) { + byte[] buffer = new byte[65536]; int n; + while ((n = input.read(buffer)) != -1) { sha.update(buffer, 0, n); md5.update(buffer, 0, n); } + } + if (!SigV4.hex(md5.digest()).equals(part.etag().replace("\"", ""))) + throw new StoreException(400, "InvalidPart", "Part ETag mismatch"); + paths.add(file); + } + ObjectStorage.Metadata result; + try (InputStream input = new PartsInput(paths)) { + result = store.put(bucket, key, input, total, SigV4.hex(sha.digest()), null, false, + readUpload(dir).contentType()); + } + remove(dir); + return result; + } + + public synchronized void abort(String id, String bucket, String key) throws IOException { + remove(upload(id, bucket, key)); + } + + private Path upload(String id, String bucket, String key) throws IOException { + if (!id.matches("[0-9a-f-]{36}")) throw new StoreException(404, "NoSuchUpload", "Upload not found"); + Path dir = root.resolve(id); + if (!Files.isDirectory(dir)) throw new StoreException(404, "NoSuchUpload", "Upload not found"); + Upload upload = readUpload(dir); + if (!upload.bucket().equals(bucket) || !upload.key().equals(key)) + throw new StoreException(404, "NoSuchUpload", "Upload not found"); + return dir; + } + private static Upload readUpload(Path dir) throws IOException { + try (var input = new DataInputStream(Files.newInputStream(dir.resolve("manifest")))) { + if (input.readInt() != MAGIC) throw new IOException("Invalid multipart upload manifest"); + Upload upload = new Upload(input.readUTF(), input.readUTF(), input.readUTF()); + if (input.read() != -1) throw new IOException("Invalid multipart upload manifest"); + return upload; + } + } + private static Path part(Path dir, int number) { return dir.resolve("part-%05d".formatted(number)); } + private void remove(Path dir) throws IOException { + long removed = 0; + try (var files = Files.list(dir)) { + for (Path file : files.toList()) { + if (file.getFileName().toString().matches("part-[0-9]{5}")) removed += Files.size(file); + Files.delete(file); + } + } + DiskStore.syncDirectory(dir); + Files.delete(dir); + DiskStore.syncDirectory(root); + staged -= removed; + active--; + } + private static MessageDigest digest(String algorithm) { + try { return MessageDigest.getInstance(algorithm); } + catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException(e); } + } + private static final class PartsInput extends InputStream { + private final Iterator parts; + private InputStream current; + PartsInput(List files) { parts = files.iterator(); } + @Override public int read() throws IOException { + byte[] one = new byte[1]; + return read(one, 0, 1) < 0 ? -1 : one[0] & 255; + } + @Override public int read(byte[] buffer, int offset, int length) throws IOException { + if (length == 0) return 0; + while (true) { + if (current == null) { + if (!parts.hasNext()) return -1; + current = Files.newInputStream(parts.next()); + } + int n = current.read(buffer, offset, length); + if (n >= 0) return n; + current.close(); current = null; + } + } + @Override public void close() throws IOException { if (current != null) current.close(); } + } +} diff --git a/src/cloud/lunarsky/store/NodeClient.java b/src/cloud/lunarsky/store/NodeClient.java new file mode 100644 index 0000000..9c7c611 --- /dev/null +++ b/src/cloud/lunarsky/store/NodeClient.java @@ -0,0 +1,147 @@ +package cloud.lunarsky.store; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.security.MessageDigest; +import java.time.Duration; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +final class NodeClient { + record Node(UUID id, UUID hostId, URI url) {} + + private static final HttpClient IDENTITY_HTTP = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(2)).build(); + private final List nodes; + private final String token; + private final String repairToken; + private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(3)).build(); + + NodeClient(List nodes, String token, String repairToken) { + if (nodes.isEmpty() || nodes.stream().map(Node::id).distinct().count() != nodes.size() || + nodes.stream().map(Node::url).distinct().count() != nodes.size()) + throw new IllegalArgumentException("Cluster node IDs and URLs must be unique"); + if (token == null || token.length() < 32) throw new IllegalArgumentException("Invalid cluster token"); + for (Node node : nodes) { + if (node.id() == null || node.hostId() == null) + throw new IllegalArgumentException("Invalid storage node identity"); + validateUrl(node.url()); + } + this.nodes = List.copyOf(nodes); + this.token = token; + this.repairToken = repairToken; + } + + int count() { return nodes.size(); } + List nodes() { return nodes; } + Node node(int index) { return nodes.get(index); } + int index(UUID id) { + for (int i = 0; i < nodes.size(); i++) if (nodes.get(i).id().equals(id)) return i; + return -1; + } + UUID faultDomain(int index, boolean testNodeDomains) { + Node node = nodes.get(index); + return testNodeDomains ? node.id() : node.hostId(); + } + + static NodeIdentity probe(URI url, String token) throws IOException { + validateUrl(url); + if (token == null || token.length() < 32) throw new IllegalArgumentException("Invalid cluster token"); + HttpRequest request = HttpRequest.newBuilder(url.resolve("/identity")) + .timeout(Duration.ofSeconds(2)).header("X-Cluster-Token", token).GET().build(); + try { + HttpResponse response = IDENTITY_HTTP.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream body = response.body()) { + if (response.statusCode() != 200) throw new IOException("Node identity request failed: " + response.statusCode()); + byte[] bytes = body.readNBytes(128); + if (bytes.length == 128) throw new IOException("Node identity response is too large"); + String[] parts = new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).trim().split(" ", -1); + if (parts.length != 2) throw new IOException("Invalid node identity response"); + return new NodeIdentity(UUID.fromString(parts[0]), UUID.fromString(parts[1])); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during node identity request", error); + } catch (IllegalArgumentException error) { + throw new IOException("Invalid node identity response", error); + } + } + + static void validateUrl(URI url) { + if (url == null || !"http".equals(url.getScheme()) || url.getHost() == null || + url.getPort() < 1 || url.getRawUserInfo() != null || + (url.getRawPath() != null && !url.getRawPath().isEmpty()) || + url.getRawQuery() != null || url.getRawFragment() != null) + throw new IllegalArgumentException("Invalid private storage node URL"); + } + + boolean availableHostsAtLeast(int required, boolean testNodeDomains) { + Set healthy = new HashSet<>(); + for (int i = 0; i < nodes.size(); i++) { + Node node = nodes.get(i); + try { + NodeIdentity actual = probe(node.url(), token); + if (actual.nodeId().equals(node.id()) && actual.hostId().equals(node.hostId())) + healthy.add(faultDomain(i, testNodeDomains)); + if (healthy.size() >= required) return true; + } catch (IOException error) { } + } + return false; + } + + void put(int index, UUID id, byte[] data, byte[] sha256) throws IOException { + put(index, id, data, sha256, false); + } + + void repair(int index, UUID id, byte[] data, byte[] sha256) throws IOException { + put(index, id, data, sha256, true); + } + + private void put(int index, UUID id, byte[] data, byte[] sha256, boolean repair) throws IOException { + Node node = nodes.get(index); + HttpRequest.Builder builder = HttpRequest.newBuilder(node.url().resolve("/segments/" + id)) + .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token) + .header("X-Cluster-Expected-Node", node.id().toString()) + .header("X-Cluster-Sha256", HexFormat.of().formatHex(sha256)); + if (repair) { + if (repairToken == null || repairToken.length() < 32) + throw new IOException("Repair authority is not available to this process"); + builder.header("X-Cluster-Repair", "true").header("X-Cluster-Repair-Token", repairToken); + } + HttpResponse response = send(builder.PUT(HttpRequest.BodyPublishers.ofByteArray(data)).build(), + HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() != 200) throw new IOException("Node " + node.id() + " rejected segment: " + response.statusCode()); + } + + byte[] get(int index, UUID id, int length, byte[] sha256) throws IOException { + if (length < 1 || length > ClusterNode.MAX_SEGMENT) throw new IOException("Invalid segment length"); + Node node = nodes.get(index); + HttpRequest request = HttpRequest.newBuilder(node.url().resolve("/segments/" + id)) + .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token) + .header("X-Cluster-Expected-Node", node.id().toString()).GET().build(); + HttpResponse response = send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream body = response.body()) { + if (response.statusCode() != 200) + throw new IOException("Node " + node.id() + " has no verified copy of segment " + id); + byte[] bytes = body.readNBytes(length + 1); + if (bytes.length != length || !MessageDigest.isEqual(SigV4.hash(bytes), sha256)) + throw new IOException("Node " + node.id() + " has no verified copy of segment " + id); + return bytes; + } + } + + private HttpResponse send(HttpRequest request, HttpResponse.BodyHandler handler) throws IOException { + try { return http.send(request, handler); } + catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during node request", error); + } + } +} diff --git a/src/cloud/lunarsky/store/NodeIdentity.java b/src/cloud/lunarsky/store/NodeIdentity.java new file mode 100644 index 0000000..59c8bb7 --- /dev/null +++ b/src/cloud/lunarsky/store/NodeIdentity.java @@ -0,0 +1,42 @@ +package cloud.lunarsky.store; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.UUID; + +record NodeIdentity(UUID nodeId, UUID hostId) { + static NodeIdentity open(Path root, UUID expectedHost) throws IOException { + Path file = root.resolve("node-identity"); + if (Files.exists(file)) { + List lines = Files.readAllLines(file, StandardCharsets.UTF_8); + if (lines.size() != 2) throw new IOException("Invalid node identity file"); + try { + NodeIdentity identity = new NodeIdentity(UUID.fromString(lines.get(0)), UUID.fromString(lines.get(1))); + if (!identity.hostId().equals(expectedHost)) + throw new IOException("Node volume belongs to a different storage host"); + return identity; + } catch (IllegalArgumentException error) { + throw new IOException("Invalid node identity file", error); + } + } + NodeIdentity identity = new NodeIdentity(UUID.randomUUID(), expectedHost); + Path temporary = Files.createTempFile(root, ".node-identity-", ".pending"); + try { + Files.writeString(temporary, identity.nodeId() + "\n" + identity.hostId() + "\n", StandardCharsets.UTF_8); + try (FileChannel channel = FileChannel.open(temporary, StandardOpenOption.WRITE)) { + channel.force(true); + } + Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE); + DiskStore.syncDirectory(root); + return identity; + } finally { + Files.deleteIfExists(temporary); + } + } +} diff --git a/src/cloud/lunarsky/store/NodeRegistry.java b/src/cloud/lunarsky/store/NodeRegistry.java new file mode 100644 index 0000000..7657b7d --- /dev/null +++ b/src/cloud/lunarsky/store/NodeRegistry.java @@ -0,0 +1,135 @@ +package cloud.lunarsky.store; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +final class NodeRegistry { + private NodeRegistry() {} + + static NodeClient.Node join(Connection connection, URI url, UUID expectedHost, String token) throws IOException { + NodeIdentity identity = NodeClient.probe(url, token); + if (!identity.hostId().equals(expectedHost)) + throw new IOException("The node reported a different physical host ID"); + try { + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.execute("SELECT pg_advisory_xact_lock(6834071092781)"); + } + try (PreparedStatement query = connection.prepareStatement( + "SELECT node_id, host_id, endpoint FROM cluster_nodes WHERE node_id=? OR endpoint=?")) { + query.setObject(1, identity.nodeId()); + query.setString(2, url.toString()); + try (ResultSet result = query.executeQuery()) { + if (result.next()) { + if (!identity.nodeId().equals(result.getObject(1)) || + !identity.hostId().equals(result.getObject(2)) || !url.toString().equals(result.getString(3))) + throw new IOException("Node ID, host ID, or endpoint conflicts with an existing registration"); + if (result.next()) throw new IOException("Conflicting node registrations"); + connection.commit(); + return new NodeClient.Node(identity.nodeId(), identity.hostId(), url); + } + } + } + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_nodes (node_id, host_id, endpoint, state) VALUES (?, ?, ?, 'active')")) { + insert.setObject(1, identity.nodeId()); + insert.setObject(2, identity.hostId()); + insert.setString(3, url.toString()); + insert.executeUpdate(); + } + connection.commit(); + return new NodeClient.Node(identity.nodeId(), identity.hostId(), url); + } catch (SQLException | IOException error) { + try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); } + if (error instanceof IOException io) throw io; + throw new IOException("Node registration failed", error); + } finally { + try { connection.setAutoCommit(true); } + catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); } + } + } + + static NodeClient load(Connection connection, List urls, String token, String repairToken) throws IOException { + if (urls.size() < 2 || urls.stream().distinct().count() != urls.size()) + throw new IllegalArgumentException("At least two distinct node URLs are required"); + try { + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.execute("SELECT pg_advisory_xact_lock(6834071092781)"); + } + Map stored = new HashMap<>(); + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT node_id, host_id, endpoint FROM cluster_nodes WHERE state <> 'retired'")) { + while (result.next()) { + URI url = URI.create(result.getString(3)); + stored.put(url.toString(), new NodeClient.Node((UUID) result.getObject(1), + (UUID) result.getObject(2), url)); + } + } + if (stored.isEmpty()) { + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_segments)")) { + result.next(); + if (result.getBoolean(1)) throw new IOException("Existing segments have no registered node identities"); + } + for (URI url : urls) { + NodeIdentity identity = NodeClient.probe(url, token); + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_nodes (node_id, host_id, endpoint, state) VALUES (?, ?, ?, 'active')")) { + insert.setObject(1, identity.nodeId()); + insert.setObject(2, identity.hostId()); + insert.setString(3, url.toString()); + insert.executeUpdate(); + } + stored.put(url.toString(), new NodeClient.Node(identity.nodeId(), identity.hostId(), url)); + } + } + List configured = new ArrayList<>(); + Set configuredIds = new HashSet<>(); + for (URI url : urls) { + NodeClient.Node node = stored.get(url.toString()); + if (node == null) throw new IOException("Unregistered storage node URL: " + url); + NodeIdentity actual = null; + try { + actual = NodeClient.probe(url, token); + } catch (IOException offline) { } + if (actual != null && (!actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId()))) + throw new IOException("Storage node identity changed at " + url); + configured.add(node); + configuredIds.add(node.id()); + } + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery( + "SELECT DISTINCT unnest(s.replica_ids) FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation")) { + while (result.next()) { + UUID id = (UUID) result.getObject(1); + if (!configuredIds.contains(id)) + throw new IOException("A live segment refers to a node missing from CLUSTER_NODES: " + id); + } + } + NodeClient nodes = new NodeClient(configured, token, repairToken); + connection.commit(); + return nodes; + } catch (SQLException | IOException | RuntimeException error) { + try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); } + if (error instanceof IOException io) throw io; + if (error instanceof SQLException sql) throw new IOException("Node registry check failed", sql); + throw (RuntimeException) error; + } finally { + try { connection.setAutoCommit(true); } + catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); } + } + } +} diff --git a/src/cloud/lunarsky/store/ObjectStorage.java b/src/cloud/lunarsky/store/ObjectStorage.java new file mode 100644 index 0000000..f0bfa81 --- /dev/null +++ b/src/cloud/lunarsky/store/ObjectStorage.java @@ -0,0 +1,26 @@ +package cloud.lunarsky.store; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +/** Storage operations shared by the local and cluster gateways. */ +interface ObjectStorage extends AutoCloseable { + record Metadata(long length, long modified, String etag, byte[] sha256, + String bucket, String key, String contentType) {} + record OpenObject(Metadata metadata, InputStream stream) implements AutoCloseable { + public void close() throws IOException { stream.close(); } + } + record ListedObject(String key, Metadata metadata) {} + record ListPage(List objects, List prefixes, String nextKey, boolean truncated) { + int keyCount() { return objects.size() + prefixes.size(); } + } + + Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, + String checksum, boolean createOnly, String contentType) throws IOException; + OpenObject open(String bucket, String key) throws IOException; + void delete(String bucket, String key) throws IOException; + ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) throws IOException; + default boolean ready() { return true; } + void close() throws IOException; +} diff --git a/src/cloud/lunarsky/store/PlacementPolicy.java b/src/cloud/lunarsky/store/PlacementPolicy.java new file mode 100644 index 0000000..bdd778e --- /dev/null +++ b/src/cloud/lunarsky/store/PlacementPolicy.java @@ -0,0 +1,38 @@ +package cloud.lunarsky.store; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +final class PlacementPolicy { + private PlacementPolicy() {} + + static List candidates(UUID segmentId, NodeClient nodes, boolean testNodeDomains) { + Map> byHost = new HashMap<>(); + for (int i = 0; i < nodes.count(); i++) + byHost.computeIfAbsent(nodes.faultDomain(i, testNodeDomains), ignored -> new ArrayList<>()).add(i); + List hosts = new ArrayList<>(byHost.keySet()); + hosts.sort(Comparator.comparingLong((UUID host) -> score(segmentId, host)).reversed()); + int longest = 0; + for (List group : byHost.values()) { + group.sort(Comparator.comparingLong((Integer index) -> score(segmentId, nodes.node(index).id())).reversed()); + longest = Math.max(longest, group.size()); + } + List order = new ArrayList<>(nodes.count()); + for (int round = 0; round < longest; round++) + for (UUID host : hosts) + if (round < byHost.get(host).size()) order.add(byHost.get(host).get(round)); + return order; + } + + private static long score(UUID segment, UUID candidate) { + ByteBuffer bytes = ByteBuffer.allocate(32); + bytes.putLong(segment.getMostSignificantBits()).putLong(segment.getLeastSignificantBits()); + bytes.putLong(candidate.getMostSignificantBits()).putLong(candidate.getLeastSignificantBits()); + return ByteBuffer.wrap(SigV4.hash(bytes.array())).getLong(); + } +} diff --git a/src/cloud/lunarsky/store/SchemaMigrator.java b/src/cloud/lunarsky/store/SchemaMigrator.java new file mode 100644 index 0000000..40ef88d --- /dev/null +++ b/src/cloud/lunarsky/store/SchemaMigrator.java @@ -0,0 +1,71 @@ +package cloud.lunarsky.store; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +final class SchemaMigrator { + private SchemaMigrator() {} + + static int prepare(Connection connection, String bucket) throws IOException { + try { + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.execute("SELECT pg_advisory_xact_lock(6834071092781)"); + statement.execute("CREATE TABLE IF NOT EXISTS cluster_schema_migrations (version integer PRIMARY KEY)"); + int version; + try (ResultSet result = statement.executeQuery("SELECT COALESCE(MAX(version), 0) FROM cluster_schema_migrations")) { + result.next(); + version = result.getInt(1); + } + if (version > 2) throw new IOException("Metadata schema is newer than this ObjectStore build"); + if (version < 1) { + statement.execute("CREATE TABLE IF NOT EXISTS cluster_usage (bucket text PRIMARY KEY, used_bytes bigint NOT NULL CHECK (used_bytes >= 0))"); + statement.execute("CREATE TABLE IF NOT EXISTS cluster_objects (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, length bigint NOT NULL, modified bigint NOT NULL, etag text NOT NULL, sha256 bytea NOT NULL, content_type text NOT NULL, PRIMARY KEY (bucket, object_key))"); + statement.execute("CREATE TABLE IF NOT EXISTS cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))"); + statement.execute("CREATE TABLE IF NOT EXISTS cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (1)"); + } + if (version < 2) { + statement.execute("ALTER TABLE cluster_segments ADD COLUMN IF NOT EXISTS replica_ids uuid[]"); + statement.execute("ALTER TABLE cluster_segments ADD COLUMN IF NOT EXISTS placement_version bigint NOT NULL DEFAULT 0"); + statement.execute("ALTER TABLE cluster_segments ADD CONSTRAINT cluster_replica_ids_required CHECK (replica_ids IS NOT NULL) NOT VALID"); + statement.execute("CREATE TABLE IF NOT EXISTS cluster_nodes (node_id uuid PRIMARY KEY, host_id uuid NOT NULL, endpoint text NOT NULL UNIQUE, legacy_index integer UNIQUE, state text NOT NULL CHECK (state IN ('joining','active','draining','offline','retired')))"); + statement.execute("CREATE TABLE IF NOT EXISTS cluster_format (singleton integer PRIMARY KEY CHECK (singleton=1), version integer NOT NULL)"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (2)"); + } + statement.execute("INSERT INTO cluster_format SELECT 1, CASE WHEN EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL) THEN 1 ELSE 2 END WHERE NOT EXISTS (SELECT 1 FROM cluster_format)"); + } + try (PreparedStatement insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, 0) ON CONFLICT DO NOTHING")) { + insert.setString(1, bucket); + insert.executeUpdate(); + } + int format; + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT version FROM cluster_format WHERE singleton=1")) { + if (!result.next()) throw new IOException("Missing cluster format marker"); + format = result.getInt(1); + } + if (format < 1 || format > 2) throw new IOException("Unsupported cluster data format " + format); + if (format == 2) { + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL)")) { + result.next(); + if (result.getBoolean(1)) throw new IOException("Cluster format has unmigrated segment replicas"); + } + } + connection.commit(); + return format; + } catch (SQLException | IOException error) { + try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); } + if (error instanceof IOException io) throw io; + throw new IOException("Metadata schema migration failed", error); + } finally { + try { connection.setAutoCommit(true); } + catch (SQLException error) { throw new IOException("Could not restore metadata connection", error); } + } + } +} diff --git a/src/cloud/lunarsky/store/StoreException.java b/src/cloud/lunarsky/store/StoreException.java index 81c4523..8905458 100644 --- a/src/cloud/lunarsky/store/StoreException.java +++ b/src/cloud/lunarsky/store/StoreException.java @@ -4,7 +4,10 @@ final class StoreException extends RuntimeException { final int status; final String code; StoreException(int status, String code, String message) { - super(message); + this(status, code, message, null); + } + StoreException(int status, String code, String message, Throwable cause) { + super(message, cause); this.status = status; this.code = code; } diff --git a/src/cloud/lunarsky/store/UnavailableMultipart.java b/src/cloud/lunarsky/store/UnavailableMultipart.java new file mode 100644 index 0000000..258e1e0 --- /dev/null +++ b/src/cloud/lunarsky/store/UnavailableMultipart.java @@ -0,0 +1,17 @@ +package cloud.lunarsky.store; + +import java.io.InputStream; +import java.util.List; + +final class UnavailableMultipart implements MultipartStorage { + private StoreException unavailable() { + return new StoreException(501, "NotImplemented", "Multipart uploads are unavailable in the local cluster prototype"); + } + @Override public String create(String bucket, String key, String contentType) { throw unavailable(); } + @Override public String putPart(String id, String bucket, String key, int number, InputStream input, + long length, String expectedHash, String checksum) { throw unavailable(); } + @Override public ObjectStorage.Metadata complete(String id, String bucket, String key, List parts) { throw unavailable(); } + @Override public void abort(String id, String bucket, String key) { throw unavailable(); } + @Override public int activeUploads() { return 0; } + @Override public long stagedBytes() { return 0; } +} diff --git a/src/cloud/lunarsky/store/Version.java b/src/cloud/lunarsky/store/Version.java new file mode 100644 index 0000000..6b66def --- /dev/null +++ b/src/cloud/lunarsky/store/Version.java @@ -0,0 +1,7 @@ +package cloud.lunarsky.store; + +final class Version { + static final String VALUE = "0.0.2"; + + private Version() {} +} diff --git a/test/cloud/lunarsky/store/CliTest.java b/test/cloud/lunarsky/store/CliTest.java new file mode 100644 index 0000000..90907ff --- /dev/null +++ b/test/cloud/lunarsky/store/CliTest.java @@ -0,0 +1,54 @@ +package cloud.lunarsky.store; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.PrintStream; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; + +public final class CliTest { + public static void main(String[] args) throws Exception { + Path root = Files.createTempDirectory("store-cli-test-"); + try { + var versionOutput = new ByteArrayOutputStream(); + if (Cli.run(new String[]{"version"}, root, new PrintStream(versionOutput), System.err) != 0 || + !versionOutput.toString().contains(Version.VALUE)) throw new AssertionError("CLI version"); + byte[] payload = "verified".getBytes(StandardCharsets.UTF_8); + try (var store = new DiskStore(root, 100, 1000)) { + store.put("objects", "example", new ByteArrayInputStream(payload), payload.length, + SigV4.hex(SigV4.hash(payload)), null, false, "text/plain"); + var output = new ByteArrayOutputStream(); + int status = Cli.run(new String[]{"status"}, root, new PrintStream(output), System.err); + if (status != 0 || !output.toString().contains("objects=1") || + !output.toString().contains("payload_bytes=8")) throw new AssertionError("CLI status"); + output.reset(); + status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), System.err); + if (status != 0 || !output.toString().contains("verified_objects=1")) + throw new AssertionError("CLI verification"); + } + String id = SigV4.hex(SigV4.hash("objects/example".getBytes(StandardCharsets.UTF_8))); + Path file = root.resolve("objects").resolve(id.substring(0, 2)).resolve(id); + int header; + try (var input = new DataInputStream(Files.newInputStream(file))) { + header = DiskStore.readRecord(input).headerLength(); + } + try (var bytes = new RandomAccessFile(file.toFile(), "rw")) { + bytes.seek(header); + bytes.write('X'); + } + var output = new ByteArrayOutputStream(); + int status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), new PrintStream(output)); + if (status != 1 || !output.toString().contains("Object checksum mismatch")) + throw new AssertionError("CLI missed corrupted payload"); + System.out.println("CLI tests passed: version, live status, verification, corruption exit code"); + } finally { + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path); + } + } + } +} diff --git a/test/cloud/lunarsky/store/ClusterIntegrationTest.java b/test/cloud/lunarsky/store/ClusterIntegrationTest.java new file mode 100644 index 0000000..61161b1 --- /dev/null +++ b/test/cloud/lunarsky/store/ClusterIntegrationTest.java @@ -0,0 +1,144 @@ +package cloud.lunarsky.store; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; + +public final class ClusterIntegrationTest { + private static final String KEY = "cluster-test/survivor"; + public static void main(String[] args) throws Exception { + Map env = System.getenv(); + String[] urls = env.get("CLUSTER_NODES").split(","); + try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"), + env.get("POSTGRES_PASSWORD"), env.get("S3_BUCKET"), + Arrays.stream(urls).map(URI::create).toList(), env.get("CLUSTER_TOKEN"), null, + 134217728, 2147483648L, + !"same-host".equals(args[0]) && "true".equals(env.get("CLUSTER_TEST_NODE_DOMAINS")))) { + String bucket = env.get("S3_BUCKET"); + byte[] stable = "acknowledged object survives node loss".getBytes(StandardCharsets.UTF_8); + switch (args[0]) { + case "basic" -> { + byte[] large = new byte[ClusterNode.MAX_SEGMENT + 37]; + for (int i = 0; i < large.length; i++) large[i] = (byte) (i * 31); + String largeKey = "cluster-test/large"; + put(store, bucket, largeKey, large, false); + try (var opened = store.open(bucket, largeKey)) { + require(opened.metadata().length() == large.length, "Wrong object length"); + require(Arrays.equals(large, opened.stream().readAllBytes()), "Multi-segment read mismatch"); + } + require(store.list(bucket, "cluster-test/", "", 10, null).keyCount() >= 1, "Listing missed object"); + try { + put(store, bucket, largeKey, large, true); + throw new AssertionError("Create-only overwrite succeeded"); + } catch (StoreException error) { require(error.status == 412, "Wrong create-only status"); } + put(store, bucket, largeKey, "replacement".getBytes(StandardCharsets.UTF_8), false); + try (var opened = store.open(bucket, largeKey)) { + require("replacement".equals(new String(opened.stream().readAllBytes(), StandardCharsets.UTF_8)), + "Overwrite was not visible"); + } + store.delete(bucket, largeKey); + try { store.open(bucket, largeKey); throw new AssertionError("Deleted object remained visible"); } + catch (StoreException error) { require(error.status == 404, "Wrong missing-object status"); } + put(store, bucket, KEY, stable, false); + require(store.ready(), "Healthy cluster is not ready"); + System.out.println("Cluster basic test passed"); + } + case "degraded" -> { + require(store.ready(), "Two available nodes should be ready"); + try (var opened = store.open(bucket, KEY)) { + require(Arrays.equals(stable, opened.stream().readAllBytes()), "Acknowledged object was lost"); + } + byte[] value = "written with one node down".getBytes(StandardCharsets.UTF_8); + put(store, bucket, "cluster-test/degraded", value, false); + try (var opened = store.open(bucket, "cluster-test/degraded")) { + require(Arrays.equals(value, opened.stream().readAllBytes()), "Degraded write was not readable"); + } + System.out.println("Cluster degraded test passed"); + } + case "quorum-lost" -> { + require(!store.ready(), "One available node must not be ready"); + try { + put(store, bucket, "cluster-test/rejected", new byte[]{1}, false); + throw new AssertionError("Write succeeded with only one node"); + } catch (StoreException error) { require(error.status == 503, "Wrong unavailable status"); } + try { store.open(bucket, "cluster-test/rejected"); throw new AssertionError("Failed write became visible"); } + catch (StoreException error) { require(error.status == 404, "Partial object became visible"); } + System.out.println("Cluster quorum-loss test passed"); + } + case "recovered" -> { + require(store.ready(), "Restarted cluster is not ready"); + try (var opened = store.open(bucket, KEY)) { + require(Arrays.equals(stable, opened.stream().readAllBytes()), "Object lost across restart"); + } + System.out.println("Cluster recovery test passed"); + } + case "concurrent" -> { + String key = "cluster-test/concurrent"; + byte[] first = "concurrent-first".getBytes(StandardCharsets.UTF_8); + byte[] second = "concurrent-second".getBytes(StandardCharsets.UTF_8); + var start = new java.util.concurrent.CountDownLatch(1); + var executor = java.util.concurrent.Executors.newFixedThreadPool(2); + try { + var a = executor.submit(() -> { + start.await(); put(store, bucket, key, first, false); return null; + }); + var b = executor.submit(() -> { + start.await(); put(store, bucket, key, second, false); return null; + }); + start.countDown(); + a.get(); b.get(); + try (var opened = store.open(bucket, key)) { + byte[] actual = opened.stream().readAllBytes(); + require(Arrays.equals(actual, first) || Arrays.equals(actual, second), + "Concurrent PUT produced a partial object"); + } + require(store.list(bucket, key, "", 10, null).keyCount() == 1, + "Concurrent PUT produced duplicate key entries"); + } finally { executor.shutdownNow(); } + System.out.println("Cluster concurrent overwrite test passed"); + } + case "same-host" -> { + require(!store.ready(), "Containers on one physical host must not form a storage quorum"); + try { + put(store, bucket, "cluster-test/same-host-rejected", new byte[]{1}, false); + throw new AssertionError("Write succeeded without two physical storage hosts"); + } catch (StoreException error) { require(error.status == 503, "Wrong same-host rejection status"); } + System.out.println("Same-host replicas correctly fail the physical-host quorum"); + } + case "joined" -> { + require(urls.length == 4, "Expansion test requires four registered nodes"); + var newNode = NodeClient.probe(URI.create(urls[3]), env.get("CLUSTER_TOKEN")); + for (int i = 0; i < 32; i++) { + String key = "cluster-test/expanded-" + i; + byte[] value = ("expanded object " + i).getBytes(StandardCharsets.UTF_8); + put(store, bucket, key, value, false); + try (var opened = store.open(bucket, key)) { + require(Arrays.equals(value, opened.stream().readAllBytes()), "Expanded object was not readable"); + } + } + try (var connection = java.sql.DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"), + env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD")); + var query = connection.prepareStatement( + "SELECT count(*) FROM cluster_segments WHERE ? = ANY(replica_ids)")) { + query.setObject(1, newNode.nodeId()); + try (var result = query.executeQuery()) { + result.next(); + require(result.getLong(1) > 0, "Joined node received no new segments"); + } + } + System.out.println("Joined node accepted new placements while previous objects stayed readable"); + } + default -> throw new IllegalArgumentException("Unknown test phase"); + } + } + } + private static void put(ClusterStore store, String bucket, String key, byte[] data, boolean createOnly) throws Exception { + store.put(bucket, key, new ByteArrayInputStream(data), data.length, SigV4.hex(SigV4.hash(data)), + null, createOnly, "application/octet-stream"); + } + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/test/cloud/lunarsky/store/ClusterMigrationTest.java b/test/cloud/lunarsky/store/ClusterMigrationTest.java new file mode 100644 index 0000000..f07454d --- /dev/null +++ b/test/cloud/lunarsky/store/ClusterMigrationTest.java @@ -0,0 +1,89 @@ +package cloud.lunarsky.store; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public final class ClusterMigrationTest { + private static final String KEY = "migration/legacy-object"; + private static final byte[] DATA = "legacy object survives stable-node migration".getBytes(StandardCharsets.UTF_8); + + public static void main(String[] args) throws Exception { + Map env = System.getenv(); + List urls = Arrays.stream(env.get("CLUSTER_NODES").split(",")) + .map(URI::create).toList(); + String token = env.get("CLUSTER_TOKEN"); + List addresses = new ArrayList<>(); + for (URI url : urls) { + NodeIdentity identity = NodeClient.probe(url, token); + addresses.add(new NodeClient.Node(identity.nodeId(), identity.hostId(), url)); + } + NodeClient nodes = new NodeClient(addresses, token, null); + String bucket = env.get("S3_BUCKET"); + if (args[0].equals("create")) { + UUID segment = UUID.randomUUID(); + byte[] hash = SigV4.hash(DATA); + for (int i = 0; i < nodes.count(); i++) nodes.put(i, segment, DATA, hash); + try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"), + env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD")); + var statement = connection.createStatement()) { + statement.execute("CREATE TABLE cluster_usage (bucket text PRIMARY KEY, used_bytes bigint NOT NULL)"); + statement.execute("CREATE TABLE cluster_objects (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, length bigint NOT NULL, modified bigint NOT NULL, etag text NOT NULL, sha256 bytea NOT NULL, content_type text NOT NULL, PRIMARY KEY (bucket, object_key))"); + statement.execute("CREATE TABLE cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))"); + statement.execute("CREATE TABLE cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))"); + try (var insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, ?)")) { + insert.setString(1, bucket); insert.setLong(2, DATA.length); insert.executeUpdate(); + } + UUID generation = UUID.randomUUID(); + try (var insert = connection.prepareStatement("INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) { + insert.setString(1, bucket); insert.setString(2, KEY); insert.setObject(3, generation); + insert.setLong(4, DATA.length); insert.setLong(5, System.currentTimeMillis()); + insert.setString(6, HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(DATA))); + insert.setBytes(7, hash); insert.setString(8, "text/plain"); insert.executeUpdate(); + } + try (var insert = connection.prepareStatement("INSERT INTO cluster_segments VALUES (?, 0, ?, ?, ?, '0,1,2')")) { + insert.setObject(1, generation); insert.setObject(2, segment); + insert.setInt(3, DATA.length); insert.setBytes(4, hash); insert.executeUpdate(); + } + } + System.out.println("Legacy cluster fixture created"); + return; + } + if (!args[0].equals("verify")) throw new IllegalArgumentException("Use create or verify"); + List reordered = new ArrayList<>(urls); + java.util.Collections.reverse(reordered); + try (ClusterStore store = new ClusterStore(env.get("POSTGRES_JDBC_URL"), env.get("POSTGRES_USER"), + env.get("POSTGRES_PASSWORD"), bucket, reordered, token, null, + 134217728, 2147483648L, true)) { + try (var opened = store.open(bucket, KEY)) { + if (!Arrays.equals(DATA, opened.stream().readAllBytes())) + throw new AssertionError("Migrated object changed after node URL reorder"); + } + store.put(bucket, "migration/new-object", new ByteArrayInputStream(DATA), DATA.length, + SigV4.hex(SigV4.hash(DATA)), null, false, "text/plain"); + } + UUID segment; + try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"), + env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD")); + var statement = connection.createStatement(); + var result = statement.executeQuery("SELECT segment_id FROM cluster_segments WHERE replicas='0,1,2' LIMIT 1")) { + if (!result.next()) throw new AssertionError("Legacy segment missing after migration"); + segment = (UUID) result.getObject(1); + } + NodeClient wrong = new NodeClient(List.of(new NodeClient.Node(addresses.get(0).id(), + addresses.get(0).hostId(), addresses.get(1).url())), token, null); + try { + wrong.get(0, segment, DATA.length, SigV4.hash(DATA)); + throw new AssertionError("Node swap was not rejected"); + } catch (java.io.IOException expected) { } + System.out.println("Stable node migration, reordered gateway config, and wrong-node rejection passed"); + } +} diff --git a/test/cloud/lunarsky/store/ClusterNodeTest.java b/test/cloud/lunarsky/store/ClusterNodeTest.java new file mode 100644 index 0000000..9f249a9 --- /dev/null +++ b/test/cloud/lunarsky/store/ClusterNodeTest.java @@ -0,0 +1,124 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.UUID; + +public final class ClusterNodeTest { + public static void main(String[] args) throws Exception { + Path root = Files.createTempDirectory("objectstore-node-"); + String token = "local-cluster-test-token-0123456789"; + String repairToken = "local-repair-test-token-0123456789"; + UUID id = UUID.randomUUID(); + UUID hostId = UUID.randomUUID(); + UUID nodeId; + byte[] value = "a durable segment".getBytes(java.nio.charset.StandardCharsets.UTF_8); + try (ClusterNode node = new ClusterNode(root, token, repairToken, hostId)) { + try { + new ClusterNode(root, token, repairToken, hostId); + throw new AssertionError("Second writer opened a locked node directory"); + } catch (IOException expected) { } + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", node::handle); + server.start(); + try { + URI uri = URI.create("http://127.0.0.1:" + server.getAddress().getPort()); + NodeIdentity identity = NodeClient.probe(uri, token); + nodeId = identity.nodeId(); + require(identity.hostId().equals(hostId), "Node reported the wrong storage host"); + NodeClient client = new NodeClient(List.of(new NodeClient.Node(identity.nodeId(), hostId, uri)), + token, repairToken); + client.put(0, id, value, SigV4.hash(value)); + client.put(0, id, value, SigV4.hash(value)); + require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))), "Roundtrip failed"); + var request = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).timeout(Duration.ofSeconds(3)) + .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString()) + .header("X-Cluster-Sha256", SigV4.hex(SigV4.hash(value))) + .PUT(HttpRequest.BodyPublishers.ofString("wrong")).build(); + var response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + require(response.statusCode() == 400, "Wrong checksum accepted"); + var unauthenticated = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).GET().build(); + require(HttpClient.newHttpClient().send(unauthenticated, HttpResponse.BodyHandlers.ofString()) + .statusCode() == 403, "Unauthenticated read accepted"); + var wrongNode = HttpRequest.newBuilder(uri.resolve("/segments/" + id)).timeout(Duration.ofSeconds(3)) + .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", UUID.randomUUID().toString()) + .GET().build(); + require(HttpClient.newHttpClient().send(wrongNode, HttpResponse.BodyHandlers.ofString()) + .statusCode() == 409, "Wrong-node request was accepted"); + Path segment = root.resolve("segments").resolve(id.toString().substring(0, 2)).resolve(id.toString()); + Files.writeString(segment, "corrupted"); + try { + client.get(0, id, value.length, SigV4.hash(value)); + throw new AssertionError("Corrupted replica passed verification"); + } catch (IOException expected) { } + NodeClient gateway = new NodeClient(List.of(new NodeClient.Node(identity.nodeId(), hostId, uri)), + token, null); + try { + gateway.repair(0, id, value, SigV4.hash(value)); + throw new AssertionError("Gateway was allowed to repair a replica"); + } catch (IOException expected) { } + var forgedRepair = HttpRequest.newBuilder(uri.resolve("/segments/" + id)) + .header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString()) + .header("X-Cluster-Repair", "true").header("X-Cluster-Sha256", SigV4.hex(SigV4.hash(value))) + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)).build(); + require(HttpClient.newHttpClient().send(forgedRepair, HttpResponse.BodyHandlers.discarding()) + .statusCode() == 403, "Shared node token was allowed to repair a replica"); + client.repair(0, id, value, SigV4.hash(value)); + require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))), + "Repair did not restore the original bytes"); + } finally { server.stop(0); } + } + Path pending = root.resolve("pending").resolve("unfinished.part"); + Files.writeString(pending, "unfinished"); + try (ClusterNode restarted = new ClusterNode(root, token, repairToken, hostId)) { + require(!Files.exists(pending), "Incomplete segment survived restart"); + Path segment = root.resolve("segments").resolve(id.toString().substring(0, 2)).resolve(id.toString()); + require(java.util.Arrays.equals(value, Files.readAllBytes(segment)), "Committed segment did not survive restart"); + } + require(NodeIdentity.open(root, hostId).nodeId().equals(nodeId), + "Node identity changed after restart"); + try { + new ClusterNode(root, token, repairToken, UUID.randomUUID()); + throw new AssertionError("Node volume accepted a changed host identity"); + } catch (IOException expected) { } + HttpServer oversized = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + oversized.createContext("/identity", exchange -> { + byte[] bytes = new byte[1024]; + exchange.sendResponseHeaders(200, 0); + exchange.getResponseBody().write(bytes); + exchange.close(); + }); + oversized.createContext("/segments/", exchange -> { + byte[] bytes = new byte[1024]; + exchange.sendResponseHeaders(200, 0); + exchange.getResponseBody().write(bytes); + exchange.close(); + }); + oversized.start(); + try { + URI uri = URI.create("http://127.0.0.1:" + oversized.getAddress().getPort()); + try { + NodeClient.probe(uri, token); + throw new AssertionError("Oversized identity response was accepted"); + } catch (IOException expected) { } + NodeClient client = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, uri)), token, null); + try { + client.get(0, id, value.length, SigV4.hash(value)); + throw new AssertionError("Oversized segment response was accepted"); + } catch (IOException expected) { } + } finally { oversized.stop(0); } + System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup"); + } + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/test/cloud/lunarsky/store/ConcurrencyTest.java b/test/cloud/lunarsky/store/ConcurrencyTest.java new file mode 100644 index 0000000..abcae89 --- /dev/null +++ b/test/cloud/lunarsky/store/ConcurrencyTest.java @@ -0,0 +1,88 @@ +package cloud.lunarsky.store; + +import java.io.ByteArrayInputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Comparator; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; + +public final class ConcurrencyTest { + private static ObjectStorage.Metadata put(DiskStore store, byte[] body) throws Exception { + return store.put("test", "shared", new ByteArrayInputStream(body), body.length, + SigV4.hex(SigV4.hash(body)), null, false, "application/octet-stream"); + } + + private static void check(DiskStore store, byte[] first, byte[] second) throws Exception { + try (var object = store.open("test", "shared")) { + byte[] body = object.stream().readAllBytes(); + if (!Arrays.equals(body, first) && !Arrays.equals(body, second)) + throw new AssertionError("Reader observed a partial object"); + if (object.metadata().length() != body.length || + !Arrays.equals(object.metadata().sha256(), SigV4.hash(body))) + throw new AssertionError("Reader observed mismatched metadata"); + } + } + + public static void main(String[] args) throws Exception { + Path root = Files.createTempDirectory("store-concurrency-test-"); + byte[] first = new byte[65536]; + byte[] second = new byte[81920]; + Arrays.fill(first, (byte) 0x35); + Arrays.fill(second, (byte) 0x67); + try (var store = new DiskStore(root, 131072, 131072); + var workers = Executors.newFixedThreadPool(4)) { + put(store, first); + var start = new CountDownLatch(1); + var done = new AtomicBoolean(false); + Future writer = workers.submit(() -> { + try { + start.await(); + for (int i = 0; i < 100; i++) { + byte[] body = i % 2 == 0 ? second : first; + var saved = put(store, body); + var listed = store.list("test", "", "", 10, null).objects(); + if (listed.size() != 1 || !listed.getFirst().metadata().etag().equals(saved.etag())) + throw new AssertionError("Acknowledged write is absent from listing"); + check(store, first, second); + } + } catch (Exception error) { + throw new RuntimeException(error); + } finally { + done.set(true); + } + }); + Future[] readers = new Future[3]; + for (int i = 0; i < readers.length; i++) { + readers[i] = workers.submit(() -> { + try { + start.await(); + while (!done.get()) check(store, first, second); + } catch (Exception error) { + throw new RuntimeException(error); + } + }); + } + start.countDown(); + writer.get(); + for (Future reader : readers) reader.get(); + store.delete("test", "shared"); + if (!store.list("test", "", "", 10, null).objects().isEmpty()) + throw new AssertionError("Acknowledged delete is present in listing"); + try { + store.open("test", "shared"); + throw new AssertionError("Acknowledged delete remained readable"); + } catch (StoreException expected) { + if (expected.status != 404) throw expected; + } + System.out.println("Concurrent storage tests passed: atomic overwrite, read and listing after acknowledgement, delete"); + } finally { + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path); + } + } + } +} diff --git a/test/cloud/lunarsky/store/HttpTest.java b/test/cloud/lunarsky/store/HttpTest.java index 4927596..dd7e24c 100644 --- a/test/cloud/lunarsky/store/HttpTest.java +++ b/test/cloud/lunarsky/store/HttpTest.java @@ -14,6 +14,8 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.Executors; public final class HttpTest { @@ -24,26 +26,34 @@ public final class HttpTest { DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC); private static HttpRequest signed(String base, String method, String key, byte[] body) { - URI uri = URI.create(base + "/objects/" + SigV4.encode(key, true)); + return signedUri(URI.create(base + "/objects/" + SigV4.encode(key, true)), method, body, Map.of()); + } + + private static HttpRequest signedUri(URI uri, String method, byte[] body, Map extra) { String host = uri.getAuthority(); String date = DATE.format(Instant.now()); String hash = SigV4.hex(SigV4.hash(body)); - String names = "host;x-amz-content-sha256;x-amz-date"; - String canonical = method + "\n" + uri.getRawPath() + "\n\n" - + "host:" + host + "\n" - + "x-amz-content-sha256:" + hash + "\n" - + "x-amz-date:" + date + "\n\n" + names + "\n" + hash; + TreeMap signed = new TreeMap<>(extra); + signed.put("host", host); + signed.put("x-amz-content-sha256", hash); + signed.put("x-amz-date", date); + String names = String.join(";", signed.keySet()); + StringBuilder canonical = new StringBuilder(method).append('\n').append(uri.getRawPath()).append('\n') + .append(SigV4.canonicalQuery(uri.getRawQuery())).append('\n'); + signed.forEach((name, value) -> canonical.append(name).append(':').append(value).append('\n')); + canonical.append('\n').append(names).append('\n').append(hash); String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" - + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8))); + + SigV4.hex(SigV4.hash(canonical.toString().getBytes(StandardCharsets.UTF_8))); String signature = SigV4.hex(SigV4.hmac( SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign)); - return HttpRequest.newBuilder(uri) + HttpRequest.Builder request = HttpRequest.newBuilder(uri) .header("x-amz-date", date) .header("x-amz-content-sha256", hash) .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" - + scope + ",SignedHeaders=" + names + ",Signature=" + signature) - .method(method, body.length == 0 + + scope + ",SignedHeaders=" + names + ",Signature=" + signature); + extra.forEach(request::header); + return request.method(method, body.length == 0 ? HttpRequest.BodyPublishers.noBody() : HttpRequest.BodyPublishers.ofByteArray(body)) .build(); @@ -77,12 +87,85 @@ public final class HttpTest { .GET().build(), HttpResponse.BodyHandlers.ofByteArray())); status(200, client.send(signed(base, "PUT", key, body), HttpResponse.BodyHandlers.ofByteArray())); + String other = "folder/stars.txt"; + status(200, client.send(signedUri(URI.create(base + "/objects/" + other), "PUT", + "stars".getBytes(StandardCharsets.UTF_8), Map.of("content-type", "text/plain")), + HttpResponse.BodyHandlers.ofByteArray())); var get = client.send(signed(base, "GET", key, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, get); if (!java.util.Arrays.equals(body, get.body())) throw new AssertionError("GET body mismatch"); if (!"application/octet-stream".equals(get.headers().firstValue("content-type").orElse(""))) throw new AssertionError("Unexpected content type"); + var typed = client.send(signed(base, "GET", other, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); + status(200, typed); + if (!"text/plain".equals(typed.headers().firstValue("content-type").orElse(""))) + throw new AssertionError("Stored content type missing"); + var partial = client.send(signedUri(URI.create(base + "/objects/" + other), "GET", + new byte[0], Map.of("range", "bytes=1-3")), HttpResponse.BodyHandlers.ofByteArray()); + status(206, partial); + if (!"tar".equals(new String(partial.body(), StandardCharsets.UTF_8)) || + !"bytes 1-3/5".equals(partial.headers().firstValue("content-range").orElse(""))) + throw new AssertionError("Range response mismatch"); + status(416, client.send(signedUri(URI.create(base + "/objects/" + other), "GET", + new byte[0], Map.of("range", "bytes=20-30")), HttpResponse.BodyHandlers.ofByteArray())); + var listed = client.send(signedUri(URI.create(base + "/objects?list-type=2&prefix=folder%2F"), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + if (listed.statusCode() != 200 || !listed.body().contains("folder/stars.txt") || + !listed.body().contains("folder/moon-☾.txt")) + throw new AssertionError("ListObjectsV2 failed: " + listed.body()); + var page = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=1"), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + if (page.statusCode() != 200 || !page.body().contains("true")) + throw new AssertionError("List pagination failed: " + page.body()); + String token = page.body().split("")[1].split("")[0]; + var next = client.send(signedUri(URI.create(base + "/objects?list-type=2&continuation-token=" + token), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + if (next.statusCode() != 200 || !next.body().contains("folder/stars.txt")) + throw new AssertionError("List continuation failed: " + next.body()); + var grouped = client.send(signedUri(URI.create(base + "/objects?list-type=2&delimiter=%2F"), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + if (grouped.statusCode() != 200 || !grouped.body().contains("folder/") || + grouped.body().contains("")) + throw new AssertionError("Delimiter listing failed: " + grouped.body()); + var encoded = client.send(signedUri(URI.create(base + "/objects?list-type=2&encoding-type=url"), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + if (encoded.statusCode() != 200 || !encoded.body().contains("folder%2Fmoon-%E2%98%BE.txt")) + throw new AssertionError("Encoded listing failed: " + encoded.body()); + var emptyPage = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=0"), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + if (emptyPage.statusCode() != 200 || !emptyPage.body().contains("0")) + throw new AssertionError("Empty list page failed: " + emptyPage.body()); + String movie = "folder/video.mp4"; + URI initiate = URI.create(base + "/objects/" + movie + "?uploads="); + var created = client.send(signedUri(initiate, "POST", new byte[0], + Map.of("content-type", "video/mp4")), HttpResponse.BodyHandlers.ofString()); + if (created.statusCode() != 200) throw new AssertionError("Multipart initiation failed: " + created.body()); + String upload = created.body().split("")[1].split("")[0]; + byte[] first = "hello ".getBytes(StandardCharsets.UTF_8); + byte[] second = "world".getBytes(StandardCharsets.UTF_8); + var partOne = client.send(signedUri(URI.create(base + "/objects/" + movie + + "?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); + var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie + + "?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); + status(200, partOne); status(200, partTwo); + String completion = "1" + + partOne.headers().firstValue("etag").orElseThrow() + + "2" + + partTwo.headers().firstValue("etag").orElseThrow() + + ""; + status(200, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload), + "POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + var assembled = client.send(signed(base, "GET", movie, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); + status(200, assembled); + if (!"hello world".equals(new String(assembled.body(), StandardCharsets.UTF_8)) || + !"video/mp4".equals(assembled.headers().firstValue("content-type").orElse(""))) + throw new AssertionError("Completed multipart object mismatch"); + var abandoned = client.send(signedUri(URI.create(base + "/objects/abandoned?uploads="), + "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + String abandonedId = abandoned.body().split("")[1].split("")[0]; + status(204, client.send(signedUri(URI.create(base + "/objects/abandoned?uploadId=" + abandonedId), + "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); var head = client.send(signed(base, "HEAD", key, new byte[0]), HttpResponse.BodyHandlers.ofByteArray()); status(200, head); @@ -91,7 +174,7 @@ public final class HttpTest { HttpResponse.BodyHandlers.ofByteArray())); status(404, client.send(signed(base, "GET", key, new byte[0]), HttpResponse.BodyHandlers.ofByteArray())); - System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE"); + System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE, MIME, ranges, listing, multipart"); } finally { server.stop(0); executor.close(); diff --git a/test/cloud/lunarsky/store/StoreTest.java b/test/cloud/lunarsky/store/StoreTest.java index 03401a7..3b6b3dc 100644 --- a/test/cloud/lunarsky/store/StoreTest.java +++ b/test/cloud/lunarsky/store/StoreTest.java @@ -2,14 +2,17 @@ package cloud.lunarsky.store; import java.nio.file.*; import java.io.*; import java.util.Arrays; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.List; public final class StoreTest { interface Operation {void run() throws Exception;} static void fails(int status,Operation operation)throws Exception{ try{operation.run();throw new AssertionError("Expected "+status);}catch(StoreException error){if(error.status!=status)throw error;} } - static DiskStore.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{ - return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only); + static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{ + return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream"); } public static void main(String[] args)throws Exception{ var headers=new com.sun.net.httpserver.Headers(); @@ -35,11 +38,13 @@ public final class StoreTest { fails(412,()->put(store,"../nested/☾",new byte[]{9},true)); fails(507,()->put(store,"second",body,true)); fails(413,()->put(store,"large",new byte[9],true)); - fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true)); - fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true)); + fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true,"application/octet-stream")); + fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true,"application/octet-stream")); fails(404,()->store.open("test","bad")); put(store,"../nested/☾",new byte[]{9},false); put(store,"empty",new byte[0],true); + if(store.list("test","","",100,null).objects().size()!=2)throw new AssertionError("List index"); + if(store.objectCount()!=2||store.legacyObjects()!=0)throw new AssertionError("Object counts"); try { new DiskStore(root,8,10); throw new AssertionError("A second process opened the same data directory"); @@ -49,9 +54,53 @@ public final class StoreTest { } try(var restarted=new DiskStore(root,8,10)){ try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");} + if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence"); restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾"); fails(404,()->restarted.open("test","../nested/☾")); + var uploads=new MultipartStore(restarted); + String upload=uploads.create("test","from-parts","text/plain"); + byte[] part={1,2,3}; + uploads.putPart(upload,"test","from-parts",1,new ByteArrayInputStream(part),part.length, + SigV4.hex(SigV4.hash(part)),null); + Files.writeString(root.resolve("pending-upload-id"),upload); } + try(var resumed=new DiskStore(root,8,10)){ + Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000"); + Files.createDirectory(unfinished); + Files.write(unfinished.resolve("manifest"),new byte[]{1,2,3}); + var uploads=new MultipartStore(resumed); + if(Files.exists(unfinished))throw new AssertionError("Unfinished multipart creation survived restart"); + String upload=Files.readString(root.resolve("pending-upload-id")); + byte[] part={1,2,3}; + String etag=SigV4.hex(MessageDigest.getInstance("MD5").digest(part)); + uploads.complete(upload,"test","from-parts",List.of(new MultipartStorage.Part(1,etag))); + try(var obj=resumed.open("test","from-parts")){ + if(!Arrays.equals(part,obj.stream().readAllBytes())||!obj.metadata().contentType().equals("text/plain")) + throw new AssertionError("Multipart restart"); + } + resumed.delete("test","from-parts"); + } + byte[] old={4,5,6}; + String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8))); + Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId); + Files.createDirectories(oldPath.getParent()); + ByteBuffer oldRecord=ByteBuffer.allocate(72+old.length).putLong(0x4c534f424a303031L) + .putLong(old.length).putLong(123456789L) + .put(MessageDigest.getInstance("MD5").digest(old)).put(SigV4.hash(old)).put(old); + Files.write(oldPath,oldRecord.array()); + try(var migrated=new DiskStore(root,8,10)){ + if(migrated.objectCount()!=2||migrated.legacyObjects()!=1)throw new AssertionError("Legacy counts"); + try(var obj=migrated.open("test","legacy")){ + if(!Arrays.equals(old,obj.stream().readAllBytes()))throw new AssertionError("Legacy read"); + } + if(migrated.list("test","","",100,null).objects().stream().anyMatch(entry->entry.key().equals("legacy"))) + throw new AssertionError("Legacy object appeared without a stored key"); + put(migrated,"legacy",old,false); + if(migrated.objectCount()!=2||migrated.legacyObjects()!=0)throw new AssertionError("Legacy count after overwrite"); + if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy"))) + throw new AssertionError("Legacy overwrite was not indexed"); + } + Files.delete(root.resolve("pending-upload-id")); String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8))); Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND); try { @@ -61,7 +110,7 @@ public final class StoreTest { if(!expected.getMessage().contains("object record"))throw expected; } try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");} - System.out.println("Java storage tests passed: roundtrip, quota, integrity, persistence, locking, corruption, delete"); + System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete"); }finally{try(var paths=Files.walk(root)){for(var p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}} } }