Release ObjectStore 0.0.2

scripts/objectstore

@@ -0,0 +1,14 @@
+#!/bin/sh
+if [ "${1:-}" = "cluster-repair" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterRepair "$@"
+fi
+if [ "${1:-}" = "cluster-migrate" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterMigrate "$@"
+fi
+if [ "${1:-}" = "cluster-join" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterJoin "$@"
+fi
+exec java -XX:MaxRAMPercentage=70 -cp /app cloud.lunarsky.store.Cli "$@"

scripts/test-cluster-http.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import datetime
+import hashlib
+import hmac
+import pathlib
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+values = dict(line.strip().split("=", 1) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()
+         if line.strip() and not line.startswith("#"))
+access = values["S3_ACCESS_KEY"]
+secret = values["S3_SECRET_KEY"]
+bucket = values.get("S3_BUCKET", "objects")
+port = values.get("CLUSTER_HOST_PORT", "9001")
+host = f"127.0.0.1:{port}"
+
+
+def sign(key, message):
+    return hmac.new(key, message.encode(), hashlib.sha256).digest()
+
+
+def request(method, path, body=b"", extra=None):
+    extra = extra or {}
+    date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    stamp = date[:8]
+    digest = hashlib.sha256(body).hexdigest()
+    uri, _, query = path.partition("?")
+    canonical = "&".join(f"{urllib.parse.quote(k, safe='~-._')}={urllib.parse.quote(v, safe='~-._')}"
+                         for k, v in sorted(urllib.parse.parse_qsl(query, keep_blank_values=True)))
+    headers = {"host": host, "x-amz-content-sha256": digest, "x-amz-date": date, **extra}
+    signed_names = ";".join(sorted(headers))
+    canonical_headers = "".join(f"{name}:{headers[name]}\n" for name in sorted(headers))
+    canonical_request = f"{method}\n{uri}\n{canonical}\n{canonical_headers}\n{signed_names}\n{digest}"
+    scope = f"{stamp}/us-east-1/s3/aws4_request"
+    to_sign = f"AWS4-HMAC-SHA256\n{date}\n{scope}\n{hashlib.sha256(canonical_request.encode()).hexdigest()}"
+    key = sign(sign(sign(sign(("AWS4" + secret).encode(), stamp), "us-east-1"), "s3"), "aws4_request")
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={access}/{scope},"
+                                f"SignedHeaders={signed_names},Signature={signature}")
+    url = f"http://{host}{path}"
+    outgoing = urllib.request.Request(url, data=body if method == "PUT" else None,
+                                      method=method, headers=headers)
+    try:
+        with urllib.request.urlopen(outgoing, timeout=30) as response:
+            return response.status, response.read(), response.headers
+    except urllib.error.HTTPError as error:
+        return error.code, error.read(), error.headers
+
+
+if len(sys.argv) > 2 and sys.argv[2] == "survivor":
+    status, content, _ = request("GET", f"/{bucket}/cluster-test/survivor")
+    assert status == 200 and content == b"acknowledged object survives node loss", (status, content)
+    print("Cluster surviving-replica HTTP read passed")
+    sys.exit(0)
+
+if len(sys.argv) > 4 and sys.argv[2] == "status":
+    key = urllib.parse.quote(sys.argv[3], safe="/")
+    expected = int(sys.argv[4])
+    status, _, _ = request("GET", f"/{bucket}/{key}")
+    assert status == expected, (status, expected)
+    print(f"Cluster GET status passed: {status}")
+    sys.exit(0)
+
+
+key = f"/{bucket}/cluster-test/http.txt"
+body = b"HTTP gateway integration test"
+status, _, _ = request("PUT", key, body)
+assert status == 200, status
+status, content, _ = request("GET", key)
+assert status == 200 and content == body, (status, content)
+status, content, _ = request("GET", key, extra={"range": "bytes=5-11"})
+assert status == 206 and content == body[5:12], (status, content)
+status, content, _ = request("HEAD", key)
+assert status == 200 and not content, status
+status, content, _ = request("GET", f"/{bucket}?list-type=2&prefix=cluster-test%2F")
+assert status == 200 and b"cluster-test/http.txt" in content, (status, content)
+status, _, _ = request("DELETE", key)
+assert status == 204, status
+status, _, _ = request("GET", key)
+assert status == 404, status
+print("Cluster HTTP tests passed: signed PUT, GET, range, HEAD, LIST, DELETE")

scripts/test-cluster.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+env_file=${1:?Usage: sh scripts/test-cluster.sh /path/to/local-cluster.env}
+host_port=${CLUSTER_HOST_PORT:-9001}
+compose() { docker compose --env-file "$env_file" -f compose.cluster.yaml "$@"; }
+restore() { compose start metadata node-a node-b >/dev/null 2>&1 || true; }
+trap restore EXIT
+compose up -d --build
+run_phase() {
+  compose exec -T gateway java --add-modules jdk.httpserver,java.net.http \
+    -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterIntegrationTest "$1"
+}
+wait_ready() {
+  attempt=0
+  until curl -fsS -o /dev/null "http://127.0.0.1:$host_port/ready" 2>/dev/null; do
+    attempt=$((attempt + 1))
+    [ "$attempt" -lt 30 ] || return 1
+    sleep 1
+  done
+}
+run_phase basic
+run_phase same-host
+run_phase concurrent
+compose stop node-a
+run_phase degraded
+compose stop node-b
+run_phase quorum-lost
+compose start node-a node-b
+wait_ready
+run_phase recovered
+segment_id=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT s.segment_id FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE o.object_key='cluster-test/survivor' LIMIT 1")
+printf '%s\n' "$segment_id" | grep -Eq '^[0-9a-f-]{36}$'
+shard=$(printf '%s' "$segment_id" | cut -c1-2)
+compose exec -T node-a sh -c 'printf corrupted > "/data/segments/$1/$2"' _ "$shard" "$segment_id"
+run_phase recovered
+compose run --rm -T repair
+expected=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT encode(s.sha256,'hex') FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE o.object_key='cluster-test/survivor' LIMIT 1")
+actual=$(compose exec -T node-a sha256sum "/data/segments/$shard/$segment_id" | cut -d' ' -f1)
+[ "$expected" = "$actual" ]
+compose stop metadata
+status=$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:$host_port/ready")
+[ "$status" = 503 ]
+compose start metadata
+wait_ready
+run_phase recovered
+python3 scripts/test-cluster-http.py "$env_file"
+compose --profile expansion up -d node-d
+compose run --rm -T --no-deps --entrypoint /usr/local/bin/objectstore gateway \
+  cluster-join http://node-d:9100 5f1447b5-3f9e-457b-8ee7-e26f0c475b5f
+export CLUSTER_NODES=http://node-a:9100,http://node-b:9100,http://node-c:9100,http://node-d:9100
+compose up -d --no-deps gateway
+wait_ready
+run_phase recovered
+run_phase joined
+echo 'Cluster failure tests passed'

scripts/test.sh

@@ -5,4 +5,7 @@ mkdir -p out/classes
 javac --release 21 --add-modules jdk.httpserver,java.net.http -d out/classes \
   src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java
 java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.StoreTest
+java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.ConcurrencyTest
 java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.HttpTest
+java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.ClusterNodeTest
+java -cp out/classes cloud.lunarsky.store.CliTest