Release ObjectStore 0.0.4

scripts/backup-cluster-metadata.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+env_file=${1:?Usage: sh scripts/backup-cluster-metadata.sh /path/to/cluster.env /path/to/metadata.dump}
+output=${2:?Usage: sh scripts/backup-cluster-metadata.sh /path/to/cluster.env /path/to/metadata.dump}
+if [ -e "$output" ]; then
+  echo "Backup destination already exists" >&2
+  exit 1
+fi
+umask 077
+temporary=$(mktemp "${output}.tmp.XXXXXX")
+trap 'rm -f "$temporary"' EXIT
+docker compose --env-file "$env_file" -f compose.cluster.yaml exec -T metadata \
+  pg_dump -U objectstore -d objectstore --format=custom --no-owner --no-acl > "$temporary"
+test -s "$temporary"
+docker compose --env-file "$env_file" -f compose.cluster.yaml exec -T metadata \
+  pg_restore --list < "$temporary" > /dev/null
+mv "$temporary" "$output"
+echo "Metadata backup written to $output"

scripts/objectstore

@@ -3,6 +3,10 @@ if [ "${1:-}" = "cluster-repair" ]; then
   shift
   exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterRepair "$@"
 fi
+if [ "${1:-}" = "cluster-gc" ]; then
+  shift
+  exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterGc "$@"
+fi
 if [ "${1:-}" = "cluster-migrate" ]; then
   shift
   exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterMigrate "$@"

scripts/test-cluster-http.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python3
 import datetime
+import base64
 import hashlib
 import hmac
 import http.client
 import pathlib
+import re
 import sys
 import urllib.parse
-import xml.etree.ElementTree as ET
+import zlib
 
 
 values = dict(line.strip().split("=", 1) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()
@@ -83,15 +85,44 @@ assert status == 204, status
 status, _, _ = request("GET", key)
 assert status == 404, status
 
+copy_source = f"/{bucket}/cluster-test/copy-source.txt"
+copy_target = f"/{bucket}/cluster-test/copied.txt"
+body = b"cluster copy and checksum test"
+crc32 = base64.b64encode(zlib.crc32(body).to_bytes(4, "big")).decode()
+md5 = base64.b64encode(hashlib.md5(body).digest()).decode()
+status, _, headers = request("PUT", copy_source, body,
+                             {"content-type": "text/plain", "content-md5": md5,
+                              "x-amz-checksum-crc32": crc32,
+                              "x-amz-sdk-checksum-algorithm": "CRC32"})
+assert status == 200 and headers["x-amz-checksum-crc32"] == crc32, status
+status, content, _ = request("PUT", copy_source, body,
+                             {"content-md5": base64.b64encode(bytes(16)).decode()})
+assert status == 400 and b"BadDigest" in content, (status, content)
+status, content, _ = request("GET", copy_source)
+assert status == 200 and content == body, (status, content)
+status, content, _ = request("PUT", copy_target, extra={"x-amz-copy-source": copy_source})
+assert status == 200 and b"<CopyObjectResult>" in content, (status, content)
+status, content, headers = request("GET", copy_target)
+assert status == 200 and content == body and headers["content-type"] == "text/plain", (status, content)
+status, _, _ = request("DELETE", copy_source)
+assert status == 204, status
+status, _, _ = request("DELETE", copy_target)
+assert status == 204, status
+
 multipart_key = f"/{bucket}/cluster-test/http-multipart.txt"
 status, content, _ = request("POST", multipart_key + "?uploads")
 assert status == 200, (status, content)
-upload_id = ET.fromstring(content).findtext("UploadId")
-assert upload_id, content
+match = re.search(rb"<UploadId>([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})</UploadId>",
+                  content[:8192])
+assert match, content
+upload_id = match.group(1).decode("ascii")
 part_etags = []
 for number, part in enumerate((b"hello ", b"world"), start=1):
-    status, _, headers = request("PUT", multipart_key + f"?partNumber={number}&uploadId={upload_id}", part)
+    checksum = base64.b64encode(hashlib.sha1(part).digest()).decode()
+    status, _, headers = request("PUT", multipart_key + f"?partNumber={number}&uploadId={upload_id}",
+                                 part, {"x-amz-checksum-sha1": checksum})
     assert status == 200, status
+    assert headers["x-amz-checksum-sha1"] == checksum, headers
     part_etags.append(headers["etag"])
 status, content, _ = request("GET", multipart_key + f"?uploadId={upload_id}&max-parts=1")
 assert status == 200 and b"<IsTruncated>true</IsTruncated>" in content, (status, content)
@@ -107,4 +138,4 @@ status, content, _ = request("GET", multipart_key)
 assert status == 200 and content == b"hello world", (status, content)
 status, content, _ = request("GET", f"/{bucket}?uploads&prefix=cluster-test%2Fhttp-multipart")
 assert status == 200 and upload_id.encode() not in content, (status, content)
-print("Cluster HTTP tests passed: signed object and multipart operations")
+print("Cluster HTTP tests passed: signed objects, copies, checksums, and multipart operations")

scripts/test-cluster.sh

@@ -3,8 +3,13 @@ set -eu
 cd "$(dirname "$0")/.."
 env_file=${1:?Usage: sh scripts/test-cluster.sh /path/to/local-cluster.env}
 host_port=${CLUSTER_HOST_PORT:-9001}
+backup_dir=
 compose() { docker compose --env-file "$env_file" -f compose.cluster.yaml "$@"; }
-restore() { compose start metadata node-a node-b >/dev/null 2>&1 || true; }
+restore() {
+  compose stop maintenance >/dev/null 2>&1 || true
+  compose start metadata node-a node-b node-c >/dev/null 2>&1 || true
+  if [ -n "$backup_dir" ]; then rm -rf "$backup_dir"; fi
+}
 trap restore EXIT
 compose up -d --build
 run_phase() {
@@ -69,4 +74,58 @@ compose up -d --no-deps gateway
 wait_ready
 run_phase recovered
 run_phase joined
+compose run --rm -T repair
+run_phase balanced
+export CLUSTER_MAINTENANCE_INTERVAL_SECONDS=1
+compose --profile automatic up -d maintenance
+node_a_id=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT node_id FROM cluster_nodes WHERE endpoint='http://node-a:9100'")
+segment_id=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT s.segment_id FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation WHERE '$node_a_id'::uuid = ANY(s.replica_ids) LIMIT 1")
+expected=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  "SELECT encode(s.sha256,'hex') FROM cluster_segments s WHERE s.segment_id='$segment_id' LIMIT 1")
+shard=$(printf '%s' "$segment_id" | cut -c1-2)
+compose exec -T node-a sh -c 'printf corrupted > "/data/segments/$1/$2"' _ "$shard" "$segment_id"
+attempt=0
+while :; do
+  actual=$(compose exec -T node-a sha256sum "/data/segments/$shard/$segment_id" | cut -d' ' -f1)
+  [ "$actual" = "$expected" ] && break
+  attempt=$((attempt + 1))
+  [ "$attempt" -lt 90 ] || { echo 'Automatic repair did not restore the replica' >&2; exit 1; }
+  sleep 1
+done
+compose stop maintenance
+export CLUSTER_GC_TEST_MODE=true CLUSTER_GC_MIN_AGE_SECONDS=0
+compose stop node-c
+if gc_refusal=$(compose run --rm -T gc --apply 2>&1); then
+  echo 'Cleanup proceeded while replicas needed repair' >&2
+  exit 1
+fi
+printf '%s\n' "$gc_refusal" | grep -q 'Refusing cleanup while live segments need repair'
+compose start node-c
+before=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  'SELECT count(*) FROM cluster_gc_candidates')
+compose run --rm -T gc
+after=$(compose exec -T metadata psql -U objectstore -d objectstore -At -c \
+  'SELECT count(*) FROM cluster_gc_candidates')
+[ "$before" = "$after" ]
+first_gc=$(compose run --rm -T gc --apply)
+printf '%s\n' "$first_gc" | grep -q '^orphan_candidates=[1-9]'
+printf '%s\n' "$first_gc" | grep -q '^segments_deleted=0$'
+second_gc=$(compose run --rm -T gc --apply)
+printf '%s\n' "$second_gc" | grep -q '^segments_deleted=[1-9]'
+run_phase recovered
+run_phase verify-expanded
+backup_dir=$(mktemp -d)
+sh scripts/backup-cluster-metadata.sh "$env_file" "$backup_dir/metadata.dump"
+compose --profile recovery up -d --wait metadata-recovery
+compose exec -T metadata-recovery pg_restore -U objectstore -d objectstore --no-owner --no-acl \
+  < "$backup_dir/metadata.dump"
+compose stop metadata
+compose run --rm -T --no-deps \
+  -e 'POSTGRES_JDBC_URL=jdbc:postgresql://metadata-recovery:5432/objectstore?connectTimeout=3&socketTimeout=10' \
+  --entrypoint java gateway --add-modules jdk.httpserver,java.net.http \
+  -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterIntegrationTest recovered
+compose start metadata
+wait_ready
 echo 'Cluster failure tests passed'