diff --git a/.env.example b/.env.example index eec2e2b..ae964f9 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,9 @@ S3_REGION=us-east-1 HOST_PORT=9000 MAX_OBJECT_BYTES=134217728 MAX_TOTAL_BYTES=2147483648 +PUBLIC_REQUESTS_PER_SECOND=0 +PUBLIC_REQUEST_BURST= +PUBLIC_BYTES_PER_SECOND=0 +PUBLIC_BYTE_BURST= +PUBLIC_MAX_IN_FLIGHT_PER_IP= +PUBLIC_TRUSTED_PROXY_IPS= diff --git a/Dockerfile b/Dockerfile index ea4898e..4694a96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,19 +4,24 @@ RUN wget -q -O /tmp/postgresql.jar https://jdbc.postgresql.org/download/postgres echo '73914527305a40cce504b0d3d90b23caf565136912d607ae8ae7c5895512332c /tmp/postgresql.jar' | sha256sum -c - COPY src ./src COPY test ./test -RUN mkdir /out && javac --release 21 --add-modules jdk.httpserver,java.net.http -d /out src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java -RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.StoreTest -RUN java --add-modules jdk.httpserver -cp /out cloud.lunarsky.store.ConcurrencyTest -RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.HttpTest -RUN java --add-modules jdk.httpserver,java.net.http -cp /out cloud.lunarsky.store.ClusterNodeTest -RUN java -cp /out cloud.lunarsky.store.CliTest +COPY lib/hash4j-0.30.0.jar /tmp/hash4j.jar +RUN echo 'd3224b113ea835ce3452097747e5209ec760cf443dbed770d6ba68a0d480178f /tmp/hash4j.jar' | sha256sum -c - +RUN mkdir /out && javac --release 21 --add-modules jdk.httpserver,java.net.http -cp /tmp/hash4j.jar -d /out src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java +RUN java --add-modules jdk.httpserver -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.StoreTest +RUN java --add-modules jdk.httpserver -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ConcurrencyTest +RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.HttpTest +RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClientLimitsTest +RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClusterNodeTest +RUN java -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.CliTest FROM eclipse-temurin:21-jre-alpine RUN addgroup -g 10001 store && adduser -D -u 10001 -G store store && mkdir /data && chown store:store /data COPY --from=build /out /app COPY --from=build /tmp/postgresql.jar /app/postgresql.jar +COPY --from=build /tmp/hash4j.jar /app/hash4j.jar +COPY lib/LICENSE.hash4j /app/LICENSE.hash4j COPY scripts/objectstore /usr/local/bin/objectstore RUN chmod 755 /usr/local/bin/objectstore USER store EXPOSE 9000 -ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.Main"] +ENTRYPOINT ["java", "-XX:MaxRAMPercentage=70", "-Dsun.net.httpserver.maxReqTime=30", "-Dsun.net.httpserver.maxRspTime=60", "-Dsun.net.httpserver.maxReqHeaders=64", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar:/app/hash4j.jar", "cloud.lunarsky.store.Main"] diff --git a/README.md b/README.md index af6c130..37a2117 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,10 @@ Source: [GitHub](https://github.com/LunarSkyOSS/ObjectStore) · [Gitea mirror](h - [S3 API support checklist](#s3-api-support-checklist) - [Tests](TESTS.md) - [Single-node setup](#single-node-setup) +- [Access keys and ACLs](#access-keys-and-acls) - [CLI and tests](#cli-and-tests) +- [Capability discovery](#capability-discovery) +- [Java client](#java-client) - [Local cluster prototype](#local-cluster-prototype) - [Migrating a local cluster](#migrating-a-local-cluster) - [Adding a cluster node](#adding-a-cluster-node) @@ -27,7 +30,7 @@ Source: [GitHub](https://github.com/LunarSkyOSS/ObjectStore) · [Gitea mirror](h ## Capability checklist -ObjectStore serves one configured bucket. +ObjectStore creates the configured default bucket at startup. Additional buckets share the configured capacity limit. - ✅ Persistent single-node storage with checksum verification - ✅ Configurable per-object and total logical size limits @@ -35,26 +38,34 @@ ObjectStore serves one configured bucket. - ✅ Local cluster prototype with stable node IDs and host-aware placement code - ✅ Opt-in automatic repair, rebalance, and guarded garbage collection in the local cluster - ✅ Metadata backup and tested restore to a separate local PostgreSQL instance +- ✅ Manual [two-machine durability and metadata-restore drill](tests/two-host/README.md) - ⬜ Production multi-server deployment and metadata failover New objects retain their content type and key. Objects written by the earlier single-node format remain readable, but cannot appear in listings until overwritten because their original keys were not stored. ## S3 API support checklist -- ✅ Header-based AWS Signature Version 4 authentication +- ✅ Header-based and presigned-query AWS Signature Version 4 authentication - ✅ `PutObject`, `GetObject`, `HeadObject`, and `DeleteObject` in both modes - ✅ Single-range GET and `ListObjectsV2` in both modes - ✅ SHA-256 payload verification and `x-amz-checksum-sha256` in both modes -- ✅ `Content-MD5` and CRC32, CRC32C, SHA-1, SHA-256, SHA-512, and MD5 checksum headers on `PutObject` and `UploadPart` +- ✅ `Content-MD5` and CRC32, CRC32C, CRC64NVME, XXHash64, XXHash3, XXHash128, SHA-1, SHA-256, SHA-512, and MD5 checksum headers on `PutObject` and `UploadPart` - ✅ `CreateMultipartUpload`, `UploadPart`, `CompleteMultipartUpload`, and `AbortMultipartUpload` in both modes - ✅ `ListParts` and `ListMultipartUploads` in both modes -- ⬜ Presigned URLs and streaming Signature V4 uploads -- ✅ `CopyObject` within the configured bucket, with `COPY` and `REPLACE` content-type behavior -- ⬜ CRC64NVME and XXHash checksums, checksum trailers, and persisted non-SHA-256 checksum metadata -- ⬜ Bucket creation and listing, object versioning, ACLs, tags, and user metadata +- ✅ Presigned URLs and signed streaming Signature V4 uploads, including signed checksum trailers +- ✅ `CreateBucket`, `HeadBucket`, `DeleteBucket`, and `ListBuckets` in both modes +- ✅ `CopyObject` across owned buckets, with `COPY` and `REPLACE` content-type and user-metadata behavior +- ✅ User metadata on object and multipart uploads; object tags on upload, copy, and the tagging subresource +- ✅ CRC64NVME and XXHash checksums, checksum trailers, and persisted object checksum metadata +- ✅ Bucket versioning with retained versions, delete markers, and version-specific reads, copies, deletes, and tags in both modes +- ✅ Basic bucket and object ACL grants, public reads, and multiple access-key identities +- ⬜ Full AWS ACL ownership controls, email grantees, and bucket policies This is an S3 API subset, not full AWS S3 compatibility. Unsupported S3 operations and Amazon-specific headers are rejected. -Checksum values are validated before an object or part is published. Non-SHA-256 checksums are returned on upload but are not stored for later reads. Copies use the existing object size limit and do not support cross-bucket or versioned sources. +Buckets use the same three-to-63-character lowercase names as the configured default bucket. The default bucket cannot be deleted through the API. Bucket creation currently accepts the empty-body request used for the configured region. Tags do not control access. +Versioning supports enabled and suspended states, historical object versions, null versions, and delete markers. Retained versions count toward the capacity limit. Deleting a specific version is permanent. Lifecycle expiration, MFA Delete, and `ListObjectVersions` delimiter grouping are not supported yet. +User metadata values currently accept printable ASCII only; non-ASCII metadata header encoding is not yet supported. +Checksums supplied with `PutObject` are validated before publication and retained across restarts, copies, and object versions. When no checksum is supplied, ObjectStore calculates and stores CRC64NVME, including for completed multipart uploads. Request `x-amz-checksum-mode: ENABLED` on `GetObject` or `HeadObject` to receive the stored checksum. `UploadPart` checksums are validated but are not yet returned by `ListParts` or combined into a multipart checksum; the completed object's CRC64NVME covers its full content. Presigned URLs use query Signature V4 with a maximum seven-day expiry. Streaming uploads support signed `aws-chunked` payloads and one signed checksum trailer. Temporary credentials and other streaming payload modes remain unsupported. Copies use the existing object size limit. ## Single-node setup @@ -70,6 +81,14 @@ Port 9000 binds to localhost. Data stays in the `object-data` Docker volume. `do The standalone defaults are 128 MiB per object and 2 GiB total. Set `MAX_OBJECT_BYTES` and `MAX_TOTAL_BYTES` in `.env` to change them. Incomplete multipart uploads consume space until aborted. +## Access keys and ACLs + +`S3_ACCESS_KEY` is the owner identity. Its secret is `S3_SECRET_KEY`. Additional keys are optional: place one `ACCESSKEY:secret` pair per line in a file mounted read-only inside the container, and set `S3_CREDENTIALS_FILE=/run/secrets/s3-credentials` in the environment file. Use a Compose override to mount an absolute host path at `/run/secrets/s3-credentials` for the `objectstore` service (or `gateway` in cluster mode). Each access key needs 16–128 alphanumeric characters and each secret at least 32 characters. A restart loads changes to that file. Keep it outside Git and protect it as a secret. Additional identities have no access until the owner grants it. + +The owner can send `x-amz-acl: public-read` or signed `x-amz-grant-*` headers on bucket creation, object uploads, copies, and multipart initiation. `GET` and `PUT ?acl` support bucket and object ACLs; a PUT accepts either signed grant headers with an empty body or an XML `AccessControlPolicy`. A grantee ID is its configured access key. Supported permissions are `READ`, `WRITE`, `READ_ACP`, `WRITE_ACP`, and `FULL_CONTROL`. The `AllUsers` group is limited to `READ`; `AuthenticatedUsers` is also recognized. Anonymous reads work only where `AllUsers` has a read grant. Replacing an object starts with a private ACL unless the new upload supplies grants; older version ACLs remain attached to their versions. + +This is a deliberately limited ACL subset. The configured owner owns every bucket and object. A `WRITE_ACP` grantee can change an object ACL only on a retained, non-null version; updates to current unversioned and null versions require the owner key. List-versions, multipart inspection, tagging, versioning controls, bucket creation/deletion, and the capability endpoint remain owner-only. There are no IAM policies, email grantees, Object Ownership modes, Block Public Access settings, or temporary credentials. Granting public bucket `READ` exposes object names through `ListObjectsV2`; granting public object `READ` exposes that object's bytes. Review those grants before exposing a gateway to the internet. + ## CLI and tests From the server shell, run the CLI inside the running container from the directory containing `compose.yaml`: @@ -82,8 +101,20 @@ docker compose exec objectstore objectstore version The startup log shows the LunarSky banner, version, and a small storage summary (`docker compose logs --tail=20 objectstore`). `status` reports object and multipart usage. `verify` also checks stored payload hashes and exits nonzero on an error. Both commands can run while the service is live; they are not a snapshot or a backup. +## Capability discovery + +ObjectStore provides a signed `GET /_objectstore/capabilities` endpoint. Use the same header-based Signature V4 authentication as the S3 API. It returns JSON with `schemaVersion: 1`, the service and software version, storage mode, supported operation names, and configured limits. The endpoint does not disclose credentials or cluster topology. The response has `Cache-Control: no-store` because limits can change after a restart. + +`operations` lists implemented API operations, not every AWS option for each operation or the caller's authorization to use them. `limits.maxObjectBytes` applies to a completed object and to each uploaded part; `limits.maxTotalBytes` is the logical storage limit; `limits.maxParts` is 10,000. Future schema version 1 responses may add fields. Clients should ignore unknown fields and treat unknown operation names as unsupported by their own implementation. This endpoint is an ObjectStore extension; a missing endpoint on another S3-compatible service does not prove that a feature is unavailable. + Run `sh scripts/test.sh` with JDK 21 to test from source. See [TESTS.md](TESTS.md) for coverage, the disposable Docker cluster suite, and the limits of those tests. +The server includes [hash4j](https://github.com/dynatrace-oss/hash4j) for streaming XXHash checksums. Its Apache-2.0 license is included at [lib/LICENSE.hash4j](lib/LICENSE.hash4j). + +## Java client + +The [JDK-only Java client](client/README.md) works with ObjectStore and other S3-compatible endpoints. It supports object transfers, listing, multipart uploads, and read-only capability discovery. An [AWT image manager](client/examples/README.md) provides a small desktop example. Run `bash client/scripts/build.sh` to produce its JAR and Javadoc locally. + ## Local cluster prototype The local cluster prototype starts three segment containers and one PostgreSQL container on the same Docker host. Copy `.env.cluster.example` to a private environment file, replace all four credentials, and run: @@ -130,6 +161,14 @@ The maintenance service is opt-in. It repairs missing or corrupt replicas and re ## Limits and safety +Public-client limits are **disabled by default**. The localhost Compose setup is unchanged. For an endpoint that accepts external clients, set one or both of `PUBLIC_REQUESTS_PER_SECOND` and `PUBLIC_BYTES_PER_SECOND` to a positive number in `.env`. The first limits requests per client IP with a token bucket; the second paces upload and download bytes through one shared per-IP budget. `PUBLIC_REQUEST_BURST` and `PUBLIC_BYTE_BURST` default to one second of their respective rates. `PUBLIC_MAX_IN_FLIGHT_PER_IP` defaults to 8 when either rate is enabled. Requests above the rate or concurrency limit receive S3 `503 SlowDown` and `Retry-After: 1`; an admitted transfer is paced rather than cut off. These are per-gateway limits, not cluster-wide quotas. The existing 16-request gateway cap and storage limits still apply. + +For example, to start with 100 requests per second and 16 MiB/s combined upload and download per IP, set `PUBLIC_REQUESTS_PER_SECOND=100` and `PUBLIC_BYTES_PER_SECOND=16777216`. Both start with a one-second burst. Adjust these numbers after measuring the actual workload; do not copy them as a universal production policy. + +ObjectStore uses the socket peer as the client IP and ignores forwarded-IP headers by default. If a reverse proxy sits between external clients and ObjectStore, set `PUBLIC_TRUSTED_PROXY_IPS` to the exact IP address that ObjectStore sees for that proxy and configure the proxy to **replace** `X-Real-IP` with its actual client IP. A request from that trusted peer without exactly one valid numeric `X-Real-IP` is rejected. Do not trust an address reachable by arbitrary clients, and preserve the original `Host` header for Signature V4. In Docker, the proxy's address as seen by the container may be a bridge address rather than `127.0.0.1`. If proxy trust is not configured, all clients behind that proxy share its budget; this is safe from header spoofing but may throttle them together. + +Enable these limits only on a deliberately public endpoint. They also apply to direct localhost storage calls to that same endpoint; leave them disabled for the local-only setup or run a separate local-only instance if local storage calls must be exempt. A direct loopback `/health` probe without a forwarded-IP header remains exempt. The byte limit is aggregate ingress plus egress for each IP, and several users behind one NAT share it. If a proxy buffers complete uploads before forwarding them, the upload byte limit controls proxy-to-ObjectStore traffic, not the client's initial upload speed; disable request buffering when end-to-end upload pacing is required. It is a fairness control, not a defense against connection floods before the Java handler runs. Put an internet-facing proxy or firewall in front of the gateway for TLS, connection limits, request timeouts, and buffering controls. Do not expose storage nodes or PostgreSQL publicly. + The local cluster has no automatic metadata failover, private-network TLS, scoped credentials, or physical host verification. Garbage collection can remove data required by an older metadata backup, so its retention guard is essential. Host UUIDs are operator labels, not proof that machines have separate power, disks, or network paths. Keep `CLUSTER_LOCAL_DEV=true` limited to local tests. The standalone cluster node binds to localhost by default. Set `NODE_BIND` only for a private test network; the Compose file binds inside its private Docker network. PostgreSQL JDBC 42.7.14 is bundled in the image with its license inside the JAR. diff --git a/TESTS.md b/TESTS.md index a42eec6..cb21368 100644 --- a/TESTS.md +++ b/TESTS.md @@ -16,11 +16,13 @@ The script compiles the source and test programs into `out/classes`, then runs: | Test | Checks | | --- | --- | -| `StoreTest` | Signature V4 test vector and tampering, local writes and reads, quotas, restart persistence, multipart recovery, legacy reads, locking, and corruption rejection. | +| `StoreTest` | Signature V4 and signed-stream vectors, CRC64NVME and XXHash reference vectors, local writes and reads, quotas, restart persistence for objects, checksums, ACLs, attributes, buckets, and versions, multipart recovery, legacy reads, locking, and corruption rejection. | | `ConcurrencyTest` | Atomic local overwrites and consistent reads, listings, and deletes during concurrent access. | -| `HttpTest` | Signed HTTP requests, object operations, same-bucket copies, checksum acceptance and rejection, ranges, listing, and multipart uploads in single-node mode. | +| `HttpTest` | Signed capability discovery, presigned URLs, streaming uploads and trailers, object and bucket operations, ACL grants with a second access key and public reads, copies, checksum persistence and rejection, ranges, listing, metadata, tags, multipart uploads, and versioning in single-node mode. | +| `ClientLimitsTest` | Disabled defaults, trusted-proxy address validation, ignored untrusted headers, per-IP request refusal, and paced response bytes. | | `ClusterNodeTest` | Node identity and locking, authenticated segment transfers, checksum rejection, repair authorization, inventory and guarded deletion, and restart cleanup. | | `CliTest` | Version, status, verification, and a nonzero result for corrupt data. | +| `ClientTest` and `MultipartClientTest` | Java client request signing, capability discovery, error handling, and multipart operations. | The script exits nonzero on failure. The test programs use temporary local directories and loopback HTTP ports; they do not use an existing ObjectStore volume. @@ -47,10 +49,14 @@ COMPOSE_PROJECT_NAME=objectstore-tests docker compose --env-file /tmp/objectstor If port 9001 is occupied, set `CLUSTER_HOST_PORT` to the same free port in both the environment file and the shell before running the script. The script reads that port from the shell; Compose reads it from the file. -The Docker suite checks signed S3 operations, same-bucket copies, upload checksums, multi-segment objects, concurrent overwrites, multipart staging and listings, completion after a gateway restart and node loss, reads and writes with a node stopped, refusal to write without a storage quorum, restart recovery, corrupt-replica repair including staged parts, metadata unavailability, and placement on a newly joined node. It then checks rebalance to the fourth node, automatic repair, garbage collection dry run and delayed deletion, and a metadata backup restored to a separate PostgreSQL instance while the primary is stopped. It also checks that containers labeled as one physical host cannot satisfy the normal host quorum. Its local-only override permits the remaining phases to use containers as separate test domains. +The Docker suite checks signed capability discovery and S3 operations, bucket creation and deletion, metadata and tags, public-read ACLs and anonymous access, copies, upload checksums, multi-segment objects, concurrent overwrites, multipart staging and listings, versioned reads and delete markers, versioned multipart completion, completion after a gateway restart and node loss, reads and writes with a node stopped, refusal to write without a storage quorum, restart recovery, corrupt-replica repair including staged parts, metadata unavailability, and placement on a newly joined node. It then checks rebalance to the fourth node, automatic repair, garbage collection dry run and delayed deletion, and a metadata backup restored to a separate PostgreSQL instance while the primary is stopped. Historical regular and multipart versions are checked after repair and cleanup. It also checks that containers labeled as one physical host cannot satisfy the normal host quorum. Its local-only override permits the remaining phases to use containers as separate test domains. `ClusterMigrationTest` is a separate legacy-format fixture and is **not** run by either test script. Do not run its `create` phase against a populated metadata database. The migration procedure is in the [README](README.md#migrating-a-local-cluster). +## Two-machine durability drill + +The [manual two-machine drill](tests/two-host/README.md) uses machine A and machine B with disposable volumes. It records acknowledged writes during a machine interruption, checks reads and write rejection with either storage machine unavailable, restarts test containers during writes, and restores a metadata backup on machine B. This drill is not part of `scripts/test.sh` because it requires two machines and a coordinated interruption. + ## What these tests do not prove - Container stops are not physical power cuts or disk failures. The automated suite does not reboot a host or test every possible crash point. diff --git a/client/.gitignore b/client/.gitignore new file mode 100644 index 0000000..849ddff --- /dev/null +++ b/client/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..991ab40 --- /dev/null +++ b/client/README.md @@ -0,0 +1,58 @@ +# ObjectStore Java client + +A small, synchronous S3-compatible client built with the JDK alone. It uses path-style URLs and AWS Signature Version 4. The code is covered by the repository's MIT license. + +## Build and test + +Requires JDK 21 or newer. No Maven, Gradle, or third-party Java libraries are needed. + +```sh +bash client/scripts/test.sh +bash client/scripts/build.sh +``` + +Run these commands from the ObjectStore repository root. The second command writes `client/dist/objectstore-client.jar` and `client/dist/javadoc/`. + +The [AWT image manager](examples/README.md) is a small drag-and-drop app for trying uploads, listing, previews, downloads, and deletes against an existing bucket. + +## Use + +```java +import cloud.lunarsky.objectstore.client.ObjectStorageClient; +import cloud.lunarsky.objectstore.client.ObjectStorageClientBuilder; +import java.net.URI; +import java.nio.charset.StandardCharsets; + +try (ObjectStorageClient storage = new ObjectStorageClientBuilder() + .endpoint(URI.create("https://storage.example.com")) + .region("us-east-1") + .credentials(accessKey, secretKey) + .build()) { + storage.putObject("photos", "hello.txt", "hello".getBytes(StandardCharsets.UTF_8), "text/plain"); + try (ObjectStorageClient.ObjectData data = storage.getObject("photos", "hello.txt")) { + data.body().transferTo(System.out); + } +} +``` + +`getObject` returns an open stream. Close it even if you do not read every byte. The client requires HTTPS unless you explicitly call `allowInsecureHttp()` for a trusted local endpoint. + +## Scope + +This version implements single-request object PUT, GET, HEAD, DELETE, ListObjectsV2 pages, standard S3 multipart initiation/part upload/list/complete/abort, unfinished-upload listing, bucket/object ACL GET and PUT, and service detection. ACLs are S3 ACL requests, not a calculation of effective permissions from IAM policies or bucket policies. An S3 service with ACLs disabled can reject them; ObjectStore implements a limited ACL subset. + +`getCapabilities()` reads ObjectStore's signed `GET /_objectstore/capabilities` response when available. It reports implemented S3 operation names, service version, storage mode, and configured limits. A listed operation means the service implements it; it does not establish that the current key may perform it or that every AWS option is supported. Older ObjectStore builds fall back to `/health` and leave unverified operations `UNKNOWN`. The response is not cached. + +For a generic S3 endpoint, successful operations on this client are recorded as `SUPPORTED`; all untested operations remain `UNKNOWN`. Call `probeReadOnlyCapabilities(bucket, existingObjectKey)` to opt into safe read probes. An S3 denial or missing object leaves that operation `UNKNOWN`, not `UNSUPPORTED`. The client never infers features from a hostname or vendor header. The older `/health` identity fallback is self-reported, not cryptographic attestation. + +Multipart upload sessions expose their bucket, key, and upload ID so a caller can persist them and inspect already uploaded parts after a restart. `uploadFileParts` sends a file in sequential parts and leaves completion to the caller. If it fails, the upload remains open for retry or explicit abort; keep the returned upload ID and do not change the source file. + +The client does not yet implement presigned URLs, credential providers, automatic retries, region redirects, or streaming uploads of unknown length. A single-request file upload hashes the file before sending it; do not modify the file while the upload runs. The client does not retry writes because a lost response can leave their outcome uncertain. + +## Errors + +- `ObjectStorageException`: an HTTP error, with status, S3 error code, request ID, and a retryability hint. +- `TransportException`: connection or I/O failure. A write may already have completed. +- `ProtocolException`: an unexpected or malformed successful response. + +The client does not include access keys, secret keys, or response bodies in exception messages. diff --git a/client/examples/README.md b/client/examples/README.md new file mode 100644 index 0000000..a8f8cdc --- /dev/null +++ b/client/examples/README.md @@ -0,0 +1,37 @@ +# Examples + +## AWT image manager + +![Image manager previewing a cat photo in the local test service](awt-images/screenshot.png) + +Cat photo in the screenshot: [IOP Publishing source image](https://ioppublishing.org/wp-content/uploads/2017/03/cat-web-cc0.jpg). + +Run from the ObjectStore repository root with JDK 21 or newer: + +```sh +bash client/examples/awt-images/run.sh +``` + +The example uses only the Java client and the JDK. Enter an S3-compatible endpoint, region, existing bucket, and access keys. Use **Connect** to verify listing access. Drag PNG, JPEG, GIF, or BMP files onto the window, or use **Add images**. Select an object to preview it; use **Download** or **Delete** to manage it. Objects go under the specified key prefix with a short random ID, so uploading the same filename does not silently replace an earlier image. The list loads 100 objects at a time. + +The example does not create a bucket. When launched directly, it keeps credentials in memory and requires explicit opt-in for plain HTTP. Use HTTP only with a trusted test service. Uploads are limited to 64 MiB per file, and previews to 12 MiB. This is a small interoperability test app, not a production asset manager. + +### Local Docker test + +Prerequisites: JDK 21 or newer, Bash, Python 3, `curl`, and a graphical desktop session. Docker must be running and accessible to your user, and `127.0.0.1:9002` must be free for the test container. + +To try the app with a separate ObjectStore service, run: + +```sh +bash client/examples/awt-images/run-test.sh +``` + +The launcher builds its local image if needed, starts a test container on `127.0.0.1:9002`, and opens the app connected to a `photos` bucket. Set `OBJECTSTORE_TEST_IMAGE` to use a different image. Closing the app leaves the container and its dedicated data volume running; run the launcher again to reconnect. Its generated test credentials are stored in the container's Docker configuration. + +When finished, remove only this example's container and volume: + +```sh +docker stop objectstore-image-example +docker rm objectstore-image-example +docker volume rm objectstore-image-example-data +``` diff --git a/client/examples/awt-images/ImageManager.java b/client/examples/awt-images/ImageManager.java new file mode 100644 index 0000000..4ebd8a0 --- /dev/null +++ b/client/examples/awt-images/ImageManager.java @@ -0,0 +1,503 @@ +import cloud.lunarsky.objectstore.client.Capabilities; +import cloud.lunarsky.objectstore.client.ObjectStorageClient; +import cloud.lunarsky.objectstore.client.ObjectStorageClientBuilder; + +import java.awt.BorderLayout; +import java.awt.Button; +import java.awt.Canvas; +import java.awt.Checkbox; +import java.awt.Color; +import java.awt.Dialog; +import java.awt.Dimension; +import java.awt.EventQueue; +import java.awt.FileDialog; +import java.awt.FlowLayout; +import java.awt.Font; +import java.awt.Frame; +import java.awt.Graphics; +import java.awt.GridLayout; +import java.awt.Label; +import java.awt.Panel; +import java.awt.TextField; +import java.awt.dnd.DnDConstants; +import java.awt.dnd.DropTarget; +import java.awt.dnd.DropTargetAdapter; +import java.awt.dnd.DropTargetDropEvent; +import java.awt.datatransfer.DataFlavor; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.ImageInputStream; +import javax.imageio.stream.MemoryCacheImageInputStream; + +/** A small, dependency-free image browser for testing an S3-compatible endpoint. */ +@SuppressWarnings("serial") +public final class ImageManager extends Frame { + private static final long MAX_UPLOAD = 64L * 1024 * 1024; + private static final int MAX_PREVIEW = 12 * 1024 * 1024; + private static final Color BACKGROUND = new Color(29, 27, 38); + private static final Color FOREGROUND = new Color(231, 223, 240); + + private final TextField endpoint = new TextField(env("S3_ENDPOINT", "http://localhost:9000")); + private final TextField region = new TextField(env("S3_REGION", "us-east-1")); + private final TextField bucket = new TextField(env("S3_BUCKET", "photos")); + private final TextField prefix = new TextField("images/"); + private final TextField access = new TextField(env("S3_ACCESS_KEY", "")); + private final TextField secret = new TextField(env("S3_SECRET_KEY", "")); + private final Checkbox allowHttp = new Checkbox("Allow HTTP for trusted tests", null, + Boolean.parseBoolean(env("S3_ALLOW_HTTP", "false"))); + private final java.awt.List images = new java.awt.List(15, false); + private final ImageCanvas preview = new ImageCanvas(); + private final Label status = new Label("Enter a bucket and credentials, then connect. Drop images to upload."); + private final Button connectButton = new Button("Connect"); + private final Button moreButton = new Button("Load more"); + private final ExecutorService io = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "objectstore-image-example"); + thread.setDaemon(true); + return thread; + }); + private final AtomicLong previewRevision = new AtomicLong(); + private final List entries = new ArrayList<>(); + private volatile ObjectStorageClient client; + private volatile String activeBucket; + private volatile String activePrefix; + private String nextToken; + + private ImageManager() { + super("ObjectStore image manager"); + setLayout(new BorderLayout(8, 8)); + setBackground(BACKGROUND); + setForeground(FOREGROUND); + setSize(920, 640); + setMinimumSize(new Dimension(650, 440)); + setLocationRelativeTo(null); + secret.setEchoChar('\u2022'); + + Panel serviceFields = new Panel(new GridLayout(2, 4, 8, 3)); + serviceFields.add(label("Endpoint")); + serviceFields.add(label("Region")); + serviceFields.add(label("Bucket")); + serviceFields.add(label("Key prefix")); + serviceFields.add(endpoint); + serviceFields.add(region); + serviceFields.add(bucket); + serviceFields.add(prefix); + Panel credentialFields = new Panel(new GridLayout(2, 2, 8, 3)); + credentialFields.add(label("Access key")); + credentialFields.add(label("Secret key")); + credentialFields.add(access); + credentialFields.add(secret); + Panel connection = new Panel(new BorderLayout(0, 5)); + connection.add(serviceFields, BorderLayout.NORTH); + connection.add(credentialFields, BorderLayout.CENTER); + + Panel actions = new Panel(new FlowLayout(FlowLayout.LEFT, 8, 4)); + Button upload = new Button("Add images..."); + Button refresh = new Button("Refresh"); + Button download = new Button("Download"); + Button delete = new Button("Delete"); + actions.add(allowHttp); + actions.add(connectButton); + actions.add(upload); + actions.add(refresh); + actions.add(moreButton); + actions.add(download); + actions.add(delete); + moreButton.setEnabled(false); + + Panel top = new Panel(new BorderLayout(0, 5)); + top.add(connection, BorderLayout.CENTER); + top.add(actions, BorderLayout.SOUTH); + add(top, BorderLayout.NORTH); + + Panel listing = new Panel(new BorderLayout(0, 5)); + listing.setPreferredSize(new Dimension(285, 400)); + listing.add(label("Images in this prefix"), BorderLayout.NORTH); + listing.add(images, BorderLayout.CENTER); + add(listing, BorderLayout.WEST); + add(preview, BorderLayout.CENTER); + add(status, BorderLayout.SOUTH); + + connectButton.addActionListener(event -> connect()); + upload.addActionListener(event -> chooseImages()); + refresh.addActionListener(event -> loadPage(true)); + moreButton.addActionListener(event -> loadPage(false)); + download.addActionListener(event -> downloadSelected()); + delete.addActionListener(event -> deleteSelected()); + images.addItemListener(event -> previewSelected()); + installDropTarget(this); + installDropTarget(preview); + installDropTarget(images); + addWindowListener(new WindowAdapter() { + @Override public void windowClosing(WindowEvent event) { + io.shutdownNow(); + ObjectStorageClient previous = client; + if (previous != null) Thread.ofVirtual().start(previous::close); + dispose(); + } + }); + if (Boolean.parseBoolean(env("S3_AUTO_CONNECT", "false"))) { + EventQueue.invokeLater(this::connect); + } + } + + private void connect() { + String url = endpoint.getText().trim(); + String location = region.getText().trim(); + String name = bucket.getText().trim(); + String folder = normalizePrefix(prefix.getText()); + String user = access.getText().trim(); + String password = secret.getText(); + boolean insecure = allowHttp.getState(); + connectButton.setEnabled(false); + setStatus("Connecting...", false); + io.execute(() -> { + ObjectStorageClient candidate = null; + try { + ObjectStorageClientBuilder builder = new ObjectStorageClientBuilder() + .endpoint(URI.create(url)).region(location).credentials(user, password) + .timeout(Duration.ofSeconds(30)); + if (insecure) builder.allowInsecureHttp(); + candidate = builder.build(); + candidate.listObjects(name, folder, null, 1); + ObjectStorageClient previous = client; + client = candidate; + activeBucket = name; + activePrefix = folder; + candidate = null; + if (previous != null) previous.close(); + String service = "S3-compatible service"; + try { + Capabilities found = client.getCapabilities(); + if (found.service() == Capabilities.ServiceKind.OBJECTSTORE) { + service = "ObjectStore" + (found.serviceVersion() == null ? "" : " " + found.serviceVersion()); + } + } catch (IOException ignored) { + // Listing already established the connection; service detection is optional. + } + String connectedService = service; + EventQueue.invokeLater(() -> { + connectButton.setEnabled(true); + setStatus("Connected to " + connectedService + ".", false); + loadPage(true); + }); + } catch (Exception error) { + if (candidate != null) candidate.close(); + showFailure("Connection failed", error); + EventQueue.invokeLater(() -> connectButton.setEnabled(true)); + } + }); + } + + private void loadPage(boolean first) { + if (client == null) { + setStatus("Connect first.", true); + return; + } + if (!first && nextToken == null) return; + String token = first ? null : nextToken; + moreButton.setEnabled(false); + setStatus(first ? "Loading images..." : "Loading more images...", false); + io.execute(() -> { + try { + ObjectStorageClient.ObjectPage page = client.listObjects(activeBucket, activePrefix, token, 100); + EventQueue.invokeLater(() -> { + if (first) { + previewRevision.incrementAndGet(); + entries.clear(); + images.removeAll(); + preview.show(null, "Select an image to preview"); + } + for (ObjectStorageClient.ObjectEntry entry : page.objects()) { + if (!isImage(entry.key())) continue; + entries.add(entry); + String name = entry.key().substring(activePrefix.length()); + images.add(name + " · " + sizeLabel(entry.size())); + } + nextToken = page.nextContinuationToken(); + moreButton.setEnabled(nextToken != null); + setStatus(entries.size() + " image(s) loaded" + + (nextToken == null ? "." : "; more available."), false); + }); + } catch (Exception error) { + showFailure("Could not list images", error); + EventQueue.invokeLater(() -> moreButton.setEnabled(token != null)); + } + }); + } + + private void chooseImages() { + FileDialog picker = new FileDialog(this, "Add images", FileDialog.LOAD); + picker.setMultipleMode(true); + picker.setVisible(true); + java.io.File[] selected = picker.getFiles(); + if (selected.length > 0) uploadImages(List.of(selected)); + } + + private void uploadImages(List files) { + if (client == null) { + setStatus("Connect first.", true); + return; + } + io.execute(() -> { + int uploaded = 0; + for (java.io.File file : files) { + try { + Path path = file.toPath(); + String name = path.getFileName().toString(); + String type = contentType(name); + if (type == null) throw new IOException("Unsupported image type"); + if (!Files.isRegularFile(path) || Files.size(path) > MAX_UPLOAD) + throw new IOException("Image must be a file of at most 64 MiB"); + String key = activePrefix + UUID.randomUUID() + "-" + name; + client.putObject(activeBucket, key, path, type); + uploaded++; + int completed = uploaded; + EventQueue.invokeLater(() -> setStatus("Uploaded " + completed + " of " + files.size() + ".", false)); + } catch (Exception error) { + showFailure("Could not upload " + file.getName(), error); + } + } + if (uploaded > 0) EventQueue.invokeLater(() -> loadPage(true)); + }); + } + + private void previewSelected() { + ObjectStorageClient.ObjectEntry entry = selectedEntry(); + long revision = previewRevision.incrementAndGet(); + if (entry == null) { + preview.show(null, "Select an image to preview"); + return; + } + preview.show(null, "Loading preview..."); + io.execute(() -> { + try (ObjectStorageClient.ObjectData data = client.getObject(activeBucket, entry.key())) { + if (data.length() > MAX_PREVIEW) throw new IOException("Preview exceeds 12 MiB"); + byte[] bytes = data.body().readNBytes(MAX_PREVIEW + 1); + if (bytes.length > MAX_PREVIEW) throw new IOException("Preview exceeds 12 MiB"); + BufferedImage image = decodePreview(bytes); + EventQueue.invokeLater(() -> { + if (previewRevision.get() == revision) preview.show(image, null); + }); + } catch (Exception error) { + EventQueue.invokeLater(() -> { + if (previewRevision.get() == revision) preview.show(null, "Preview unavailable"); + }); + showFailure("Could not preview image", error); + } + }); + } + + private void downloadSelected() { + ObjectStorageClient.ObjectEntry entry = selectedEntry(); + if (entry == null) { + setStatus("Select an image first.", true); + return; + } + FileDialog picker = new FileDialog(this, "Save image", FileDialog.SAVE); + picker.setFile(Path.of(entry.key()).getFileName().toString()); + picker.setVisible(true); + if (picker.getFile() == null) return; + Path destination = Path.of(picker.getDirectory(), picker.getFile()); + if (Files.exists(destination) && !confirm("Replace this file?", destination.toString())) return; + io.execute(() -> { + Path temporary = null; + try { + Path parent = destination.toAbsolutePath().getParent(); + temporary = Files.createTempFile(parent, ".objectstore-image-", ".part"); + try (ObjectStorageClient.ObjectData data = client.getObject(activeBucket, entry.key())) { + Files.copy(data.body(), temporary, StandardCopyOption.REPLACE_EXISTING); + } + Files.move(temporary, destination, StandardCopyOption.REPLACE_EXISTING); + EventQueue.invokeLater(() -> setStatus("Saved " + destination.getFileName() + ".", false)); + } catch (Exception error) { + showFailure("Download failed", error); + } finally { + if (temporary != null) { + try { Files.deleteIfExists(temporary); } catch (IOException ignored) { } + } + } + }); + } + + private void deleteSelected() { + ObjectStorageClient.ObjectEntry entry = selectedEntry(); + if (entry == null) { + setStatus("Select an image first.", true); + return; + } + if (!confirm("Delete this image?", entry.key())) return; + io.execute(() -> { + try { + client.deleteObject(activeBucket, entry.key()); + EventQueue.invokeLater(() -> { + previewRevision.incrementAndGet(); + loadPage(true); + }); + } catch (Exception error) { showFailure("Delete failed", error); } + }); + } + + private ObjectStorageClient.ObjectEntry selectedEntry() { + int index = images.getSelectedIndex(); + return index < 0 || index >= entries.size() ? null : entries.get(index); + } + + private void installDropTarget(java.awt.Component target) { + new DropTarget(target, DnDConstants.ACTION_COPY, new DropTargetAdapter() { + @Override public void drop(DropTargetDropEvent event) { + if (!event.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) { + event.rejectDrop(); + return; + } + try { + event.acceptDrop(DnDConstants.ACTION_COPY); + Object value = event.getTransferable().getTransferData(DataFlavor.javaFileListFlavor); + List dropped = (List) value; + List files = new ArrayList<>(); + for (Object item : dropped) if (item instanceof java.io.File file) files.add(file); + event.dropComplete(true); + EventQueue.invokeLater(() -> uploadImages(files)); + } catch (Exception error) { + event.dropComplete(false); + showFailure("Drop failed", error); + } + } + }, true); + } + + private boolean confirm(String title, String detail) { + Dialog dialog = new Dialog(this, title, true); + dialog.setLayout(new BorderLayout(12, 12)); + dialog.add(new Label(detail), BorderLayout.CENTER); + Panel buttons = new Panel(new FlowLayout(FlowLayout.RIGHT)); + boolean[] accepted = { false }; + Button cancel = new Button("Cancel"); + Button proceed = new Button("Continue"); + cancel.addActionListener(event -> dialog.dispose()); + proceed.addActionListener(event -> { accepted[0] = true; dialog.dispose(); }); + buttons.add(cancel); + buttons.add(proceed); + dialog.add(buttons, BorderLayout.SOUTH); + dialog.setSize(490, 120); + dialog.setLocationRelativeTo(this); + dialog.setVisible(true); + return accepted[0]; + } + + private void showFailure(String action, Exception error) { + EventQueue.invokeLater(() -> setStatus(action + ": " + error.getMessage(), true)); + } + + private void setStatus(String message, boolean failed) { + status.setForeground(failed ? new Color(245, 143, 157) : FOREGROUND); + status.setText(message); + } + + private static Label label(String text) { + return new Label(text); + } + + private static String normalizePrefix(String value) { + String cleaned = value.trim().replace('\\', '/'); + while (cleaned.startsWith("/")) cleaned = cleaned.substring(1); + return cleaned.isEmpty() || cleaned.endsWith("/") ? cleaned : cleaned + "/"; + } + + private static String contentType(String name) { + String lower = name.toLowerCase(Locale.ROOT); + if (lower.endsWith(".png")) return "image/png"; + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"; + if (lower.endsWith(".gif")) return "image/gif"; + if (lower.endsWith(".bmp")) return "image/bmp"; + return null; + } + + private static boolean isImage(String name) { + return contentType(name) != null; + } + + private static String sizeLabel(long bytes) { + return bytes < 1024 ? bytes + " B" : String.format(Locale.ROOT, "%.1f KiB", bytes / 1024.0); + } + + private static BufferedImage decodePreview(byte[] bytes) throws IOException { + try (ImageInputStream stream = new MemoryCacheImageInputStream(new ByteArrayInputStream(bytes))) { + var readers = ImageIO.getImageReaders(stream); + if (!readers.hasNext()) throw new IOException("Unsupported image data"); + ImageReader reader = readers.next(); + try { + reader.setInput(stream, true, true); + int width = reader.getWidth(0); + int height = reader.getHeight(0); + if (width < 1 || height < 1 || width > 16000 || height > 16000 || + (long) width * height > 40_000_000) + throw new IOException("Image dimensions are too large for preview"); + int step = Math.max(1, (Math.max(width, height) + 1199) / 1200); + var parameters = reader.getDefaultReadParam(); + parameters.setSourceSubsampling(step, step, 0, 0); + return reader.read(0, parameters); + } finally { reader.dispose(); } + } + } + + private static String env(String name, String fallback) { + String value = System.getenv(name); + return value == null ? fallback : value; + } + + @SuppressWarnings("serial") + private static final class ImageCanvas extends Canvas { + private BufferedImage image; + private String message = "Select an image to preview"; + + private ImageCanvas() { + setBackground(new Color(21, 20, 29)); + setForeground(FOREGROUND); + setFont(new Font(Font.SANS_SERIF, Font.PLAIN, 16)); + } + + private void show(BufferedImage value, String text) { + image = value; + message = text; + repaint(); + } + + @Override public void paint(Graphics graphics) { + int width = getWidth(); + int height = getHeight(); + if (image == null) { + graphics.setColor(FOREGROUND); + graphics.drawString(message, 20, Math.max(35, height / 2)); + return; + } + double scale = Math.min((width - 24.0) / image.getWidth(), + (height - 24.0) / image.getHeight()); + scale = Math.max(0.01, Math.min(scale, 1.0)); + int drawWidth = (int) Math.round(image.getWidth() * scale); + int drawHeight = (int) Math.round(image.getHeight() * scale); + graphics.drawImage(image, (width - drawWidth) / 2, (height - drawHeight) / 2, + drawWidth, drawHeight, null); + } + } + + public static void main(String[] args) { + EventQueue.invokeLater(() -> new ImageManager().setVisible(true)); + } +} diff --git a/client/examples/awt-images/run-test.sh b/client/examples/awt-images/run-test.sh new file mode 100755 index 0000000..f308d8e --- /dev/null +++ b/client/examples/awt-images/run-test.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +client_dir="$(cd "$(dirname "$0")/../.." && pwd)" +project_dir="$(cd "$client_dir/.." && pwd)" +container="objectstore-image-example" +volume="objectstore-image-example-data" +image="${OBJECTSTORE_TEST_IMAGE:-objectstore-image-example:local}" + +if ! docker container inspect "$container" >/dev/null 2>&1; then + if ! docker image inspect "$image" >/dev/null 2>&1; then + docker build --tag "$image" "$project_dir" + fi + export S3_ACCESS_KEY="ImageExampleTest1" + export S3_SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" + docker volume create "$volume" >/dev/null + docker run --detach --name "$container" \ + --publish 127.0.0.1:9002:9000 \ + --volume "$volume:/data" \ + --env S3_ACCESS_KEY --env S3_SECRET_KEY \ + --env S3_BUCKET=photos --env S3_REGION=us-east-1 \ + --env MAX_OBJECT_BYTES=67108864 --env MAX_TOTAL_BYTES=1073741824 \ + "$image" >/dev/null +else + while IFS='=' read -r key value; do + case "$key" in + S3_ACCESS_KEY) export S3_ACCESS_KEY="$value" ;; + S3_SECRET_KEY) export S3_SECRET_KEY="$value" ;; + esac + done < <(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$container") + if [[ "$(docker inspect --format '{{.State.Running}}' "$container")" != true ]]; then + docker start "$container" >/dev/null + fi +fi + +for attempt in {1..40}; do + if curl --silent --fail --output /dev/null http://127.0.0.1:9002/health; then + break + fi + sleep 0.25 +done +if ! curl --silent --fail --output /dev/null http://127.0.0.1:9002/health; then + echo "The isolated ObjectStore container did not become healthy; check docker logs $container." >&2 + exit 1 +fi +export S3_ENDPOINT=http://127.0.0.1:9002 +export S3_REGION=us-east-1 +export S3_BUCKET=photos +export S3_ALLOW_HTTP=true +export S3_AUTO_CONNECT=true +exec "$client_dir/examples/awt-images/run.sh" diff --git a/client/examples/awt-images/run.sh b/client/examples/awt-images/run.sh new file mode 100755 index 0000000..cd8572f --- /dev/null +++ b/client/examples/awt-images/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +client_dir="$(cd "$(dirname "$0")/../.." && pwd)" +if [[ ! -f "$client_dir/dist/objectstore-client.jar" ]]; then + bash "$client_dir/scripts/build.sh" +fi +classes="$client_dir/dist/examples/awt-images" +mkdir -p "$classes" +javac --release 21 -cp "$client_dir/dist/objectstore-client.jar" \ + -d "$classes" "$client_dir/examples/awt-images/ImageManager.java" +exec java -cp "$classes:$client_dir/dist/objectstore-client.jar" ImageManager diff --git a/client/examples/awt-images/screenshot.png b/client/examples/awt-images/screenshot.png new file mode 100644 index 0000000..da39dc7 Binary files /dev/null and b/client/examples/awt-images/screenshot.png differ diff --git a/client/scripts/build.sh b/client/scripts/build.sh new file mode 100644 index 0000000..2869a31 --- /dev/null +++ b/client/scripts/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +rm -rf dist/classes +mkdir -p dist/classes +find src/main/java -name '*.java' -print0 | + xargs -0 javac --release 21 -d dist/classes +mkdir -p dist/classes/META-INF +cp ../LICENSE dist/classes/META-INF/LICENSE +jar --create --file dist/objectstore-client.jar -C dist/classes . +javadoc --release 21 -quiet -Xdoclint:reference,syntax,html -d dist/javadoc \ + $(find src/main/java -name '*.java' -print) +echo "Built dist/objectstore-client.jar" diff --git a/client/scripts/test.sh b/client/scripts/test.sh new file mode 100644 index 0000000..50fb9f7 --- /dev/null +++ b/client/scripts/test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +build_dir="$(mktemp -d)" +trap 'rm -rf "$build_dir"' EXIT +find src/main/java src/test/java -name '*.java' -print0 | + xargs -0 javac --release 21 -d "$build_dir" +java -cp "$build_dir" cloud.lunarsky.objectstore.client.ClientTest +java -cp "$build_dir" cloud.lunarsky.objectstore.client.MultipartClientTest diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/AclPolicy.java b/client/src/main/java/cloud/lunarsky/objectstore/client/AclPolicy.java new file mode 100644 index 0000000..7f3070e --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/AclPolicy.java @@ -0,0 +1,27 @@ +package cloud.lunarsky.objectstore.client; + +import java.util.List; +import java.util.Objects; + +/** A bucket or object ACL. This is not an effective-permissions calculation. */ +public record AclPolicy(String ownerId, List grants) { + /** A canonical user or predefined S3 group. */ + public enum GranteeType { CANONICAL_USER, GROUP } + + /** An S3 ACL permission. */ + public enum Permission { READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL } + + /** One ACL grant to a canonical user ID or S3 group URI. */ + public record Grant(GranteeType type, String grantee, Permission permission) { + public Grant { + Objects.requireNonNull(type, "type"); + if (grantee == null || grantee.isBlank()) throw new IllegalArgumentException("grantee is required"); + Objects.requireNonNull(permission, "permission"); + } + } + + public AclPolicy { + if (ownerId == null || ownerId.isBlank()) throw new IllegalArgumentException("owner ID is required"); + grants = List.copyOf(grants); + } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/Capabilities.java b/client/src/main/java/cloud/lunarsky/objectstore/client/Capabilities.java new file mode 100644 index 0000000..f7b3e35 --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/Capabilities.java @@ -0,0 +1,43 @@ +package cloud.lunarsky.objectstore.client; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** Reported or observed operation support, separate from the caller's authorization. */ +public record Capabilities(ServiceKind service, Map operations, Map limits, + String serviceVersion, String storageMode, boolean completeOperationInventory) { + private static final Set KNOWN_OPERATIONS = Set.of( + "ListBuckets", "CreateBucket", "HeadBucket", "DeleteBucket", "ListObjectsV2", + "PutObject", "GetObject", "HeadObject", "DeleteObject", "CopyObject", + "GetObjectTagging", "PutObjectTagging", "DeleteObjectTagging", + "CreateMultipartUpload", "UploadPart", "ListParts", "CompleteMultipartUpload", + "AbortMultipartUpload", "ListMultipartUploads", "GetBucketVersioning", + "PutBucketVersioning", "ListObjectVersions", "GetBucketAcl", "PutBucketAcl", + "GetObjectAcl", "PutObjectAcl"); + + /** Identifies a self-reported ObjectStore service or an unrecognized S3-compatible service. */ + public enum ServiceKind { OBJECTSTORE, UNKNOWN_S3 } + + /** Three-state feature support; a generic S3 server cannot be inferred from its hostname. */ + public enum Support { SUPPORTED, UNSUPPORTED, UNKNOWN } + + public Capabilities { + Objects.requireNonNull(service, "service"); + operations = Map.copyOf(operations); + limits = Map.copyOf(limits); + } + + /** Constructs a result without a server manifest or configured limits. */ + public Capabilities(ServiceKind service, Map operations) { + this(service, operations, Map.of(), null, null, false); + } + + /** Returns support for a named operation. A missing manifest or denied probe stays UNKNOWN. */ + public Support support(String operation) { + Support observed = operations.get(operation); + if (observed != null) return observed; + if (completeOperationInventory && KNOWN_OPERATIONS.contains(operation)) return Support.UNSUPPORTED; + return Support.UNKNOWN; + } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/Json.java b/client/src/main/java/cloud/lunarsky/objectstore/client/Json.java new file mode 100644 index 0000000..2b9d5e4 --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/Json.java @@ -0,0 +1,194 @@ +package cloud.lunarsky.objectstore.client; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +final class Json { + private Json() { } + + static Object parse(byte[] bytes) throws ProtocolException { + String source; + try { source = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes)).toString(); } + catch (CharacterCodingException e) { throw new ProtocolException("Capability JSON is not UTF-8", e); } + Parser parser = new Parser(source); + Object value = parser.value(0); + parser.space(); + if (parser.index != source.length()) throw new ProtocolException("Trailing capability JSON data"); + return value; + } + + @SuppressWarnings("unchecked") + static Map object(Object value, String name) throws ProtocolException { + if (!(value instanceof Map)) throw new ProtocolException(name + " must be an object"); + return (Map) value; + } + + static List array(Object value, String name) throws ProtocolException { + if (!(value instanceof List list)) throw new ProtocolException(name + " must be an array"); + return list; + } + + static String string(Object value, String name) throws ProtocolException { + if (!(value instanceof String text) || text.isBlank()) + throw new ProtocolException(name + " must be a nonempty string"); + return text; + } + + static long integer(Object value, String name) throws ProtocolException { + if (!(value instanceof BigDecimal number)) throw new ProtocolException(name + " must be an integer"); + try { return number.longValueExact(); } + catch (ArithmeticException e) { throw new ProtocolException(name + " is out of range", e); } + } + + private static final class Parser { + private final String source; + private int index; + + private Parser(String source) { this.source = source; } + + private void space() { + while (index < source.length() && (source.charAt(index) == ' ' || source.charAt(index) == '\n' || + source.charAt(index) == '\r' || source.charAt(index) == '\t')) index++; + } + + private Object value(int depth) throws ProtocolException { + if (depth > 16) throw new ProtocolException("Capability JSON is too deeply nested"); + space(); + if (index >= source.length()) throw new ProtocolException("Incomplete capability JSON"); + return switch (source.charAt(index)) { + case '{' -> object(depth + 1); + case '[' -> array(depth + 1); + case '"' -> string(); + case 't' -> literal("true", Boolean.TRUE); + case 'f' -> literal("false", Boolean.FALSE); + case 'n' -> literal("null", null); + default -> number(); + }; + } + + private Map object(int depth) throws ProtocolException { + index++; + space(); + Map result = new HashMap<>(); + if (take('}')) return result; + do { + space(); + if (index >= source.length() || source.charAt(index) != '"') + throw new ProtocolException("Capability JSON object key is missing"); + String key = string(); + space(); + require(':'); + if (result.containsKey(key)) throw new ProtocolException("Duplicate capability JSON key"); + result.put(key, value(depth)); + if (result.size() > 256) throw new ProtocolException("Capability JSON object is too large"); + space(); + if (take('}')) return result; + require(','); + } while (true); + } + + private List array(int depth) throws ProtocolException { + index++; + space(); + List result = new ArrayList<>(); + if (take(']')) return result; + do { + result.add(value(depth)); + if (result.size() > 2048) throw new ProtocolException("Capability JSON array is too large"); + space(); + if (take(']')) return result; + require(','); + } while (true); + } + + private String string() throws ProtocolException { + index++; + StringBuilder result = new StringBuilder(); + while (index < source.length()) { + char current = source.charAt(index++); + if (current == '"') { + for (int i = 0; i < result.length(); i++) { + char unit = result.charAt(i); + if (Character.isHighSurrogate(unit)) { + if (++i >= result.length() || !Character.isLowSurrogate(result.charAt(i))) + throw new ProtocolException("Invalid JSON Unicode surrogate"); + } else if (Character.isLowSurrogate(unit)) + throw new ProtocolException("Invalid JSON Unicode surrogate"); + } + return result.toString(); + } + if (current < 0x20) throw new ProtocolException("Unescaped control character in JSON string"); + if (current != '\\') { result.append(current); continue; } + if (index >= source.length()) throw new ProtocolException("Incomplete JSON escape"); + char escaped = source.charAt(index++); + switch (escaped) { + case '"', '\\', '/' -> result.append(escaped); + case 'b' -> result.append('\b'); + case 'f' -> result.append('\f'); + case 'n' -> result.append('\n'); + case 'r' -> result.append('\r'); + case 't' -> result.append('\t'); + case 'u' -> { + if (index + 4 > source.length()) throw new ProtocolException("Incomplete Unicode escape"); + int unit = 0; + for (int i = 0; i < 4; i++) { + int digit = Character.digit(source.charAt(index++), 16); + if (digit < 0) throw new ProtocolException("Invalid Unicode escape"); + unit = unit * 16 + digit; + } + result.append((char) unit); + } + default -> throw new ProtocolException("Invalid JSON escape"); + } + } + throw new ProtocolException("Unterminated JSON string"); + } + + private Object literal(String expected, Object value) throws ProtocolException { + if (!source.startsWith(expected, index)) throw new ProtocolException("Invalid JSON literal"); + index += expected.length(); + return value; + } + + private BigDecimal number() throws ProtocolException { + int start = index; + if (take('-') && index >= source.length()) throw new ProtocolException("Invalid JSON number"); + if (take('0')) { + if (index < source.length() && Character.isDigit(source.charAt(index))) + throw new ProtocolException("Invalid JSON number"); + } else { + if (index >= source.length() || source.charAt(index) < '1' || source.charAt(index) > '9') + throw new ProtocolException("Invalid JSON number"); + while (index < source.length() && source.charAt(index) >= '0' && source.charAt(index) <= '9') index++; + } + if (take('.')) { + int first = index; + while (index < source.length() && source.charAt(index) >= '0' && source.charAt(index) <= '9') index++; + if (first == index) throw new ProtocolException("Invalid JSON fraction"); + } + if (take('e') || take('E')) { + if (!take('+')) take('-'); + int first = index; + while (index < source.length() && source.charAt(index) >= '0' && source.charAt(index) <= '9') index++; + if (first == index) throw new ProtocolException("Invalid JSON exponent"); + } + try { return new BigDecimal(source.substring(start, index)); } + catch (NumberFormatException e) { throw new ProtocolException("Invalid JSON number", e); } + } + + private boolean take(char value) { + if (index < source.length() && source.charAt(index) == value) { index++; return true; } + return false; + } + + private void require(char value) throws ProtocolException { + if (!take(value)) throw new ProtocolException("Expected '" + value + "' in capability JSON"); + } + } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClient.java b/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClient.java new file mode 100644 index 0000000..7975b07 --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClient.java @@ -0,0 +1,633 @@ +package cloud.lunarsky.objectstore.client; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLDecoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.w3c.dom.Element; + +/** A synchronous, path-style S3 client. Instances are safe to share between threads. */ +public final class ObjectStorageClient implements AutoCloseable { + private static final int MAX_XML = 4 * 1024 * 1024; + private static final int MAX_ERROR = 64 * 1024; + private static final String EMPTY_HASH = SigV4.hash(new byte[0]); + private final URI endpoint; + private final String region; + private final String accessKey; + private final String secretKey; + private final Duration timeout; + private final Clock clock; + private final HttpClient http; + private final Set observedOperations = ConcurrentHashMap.newKeySet(); + + ObjectStorageClient(URI endpoint, String region, String accessKey, String secretKey, + Duration timeout, Clock clock) { + this.endpoint = endpoint; + this.region = region; + this.accessKey = accessKey; + this.secretKey = secretKey; + this.timeout = timeout; + this.clock = clock; + this.http = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(timeout).version(HttpClient.Version.HTTP_1_1).build(); + } + + /** Returns one open object stream. The caller must close the returned result. */ + public ObjectData getObject(String bucket, String key) throws IOException { + HttpResponse response = send("GET", objectPath(bucket, key), Map.of(), Map.of(), + HttpRequest.BodyPublishers.noBody(), EMPTY_HASH); + if (!successful(response)) { + try (InputStream ignored = response.body()) { throw failure(response); } + } + observedOperations.add("GetObject"); + return new ObjectData(response.body(), length(response), response.headers().firstValue("content-type").orElse(null), + response.headers().firstValue("etag").orElse(null)); + } + + /** Returns object headers without downloading content. */ + public ObjectMetadata headObject(String bucket, String key) throws IOException { + HttpResponse response = send("HEAD", objectPath(bucket, key), Map.of(), Map.of(), + HttpRequest.BodyPublishers.noBody(), EMPTY_HASH); + try (InputStream ignored = response.body()) { + if (!successful(response)) throw failure(response); + observedOperations.add("HeadObject"); + return new ObjectMetadata(length(response), response.headers().firstValue("content-type").orElse(null), + response.headers().firstValue("etag").orElse(null)); + } + } + + /** Uploads an immutable byte array as one object. No automatic write retry is performed. */ + public void putObject(String bucket, String key, byte[] content, String contentType) throws IOException { + Objects.requireNonNull(content, "content"); + put(bucket, key, HttpRequest.BodyPublishers.ofByteArray(content), SigV4.hash(content), contentType); + } + + /** Uploads a file. Do not change the file between hashing and completion of the upload. */ + public void putObject(String bucket, String key, Path file, String contentType) throws IOException { + Objects.requireNonNull(file, "file"); + String hash; + try (InputStream input = Files.newInputStream(file)) { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] buffer = new byte[65536]; + for (int count; (count = input.read(buffer)) >= 0; ) digest.update(buffer, 0, count); + hash = java.util.HexFormat.of().formatHex(digest.digest()); + } catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); } + put(bucket, key, HttpRequest.BodyPublishers.ofFile(file), hash, contentType); + } + + private void put(String bucket, String key, HttpRequest.BodyPublisher body, String hash, + String contentType) throws IOException { + Map headers = contentType == null ? Map.of() : Map.of("content-type", contentType); + HttpResponse response = send("PUT", objectPath(bucket, key), Map.of(), headers, body, hash); + try (InputStream ignored = response.body()) { + if (!successful(response)) throw failure(response); + observedOperations.add("PutObject"); + } + } + + /** Deletes the current object. A versioned backend may create a delete marker. */ + public void deleteObject(String bucket, String key) throws IOException { + HttpResponse response = send("DELETE", objectPath(bucket, key), Map.of(), Map.of(), + HttpRequest.BodyPublishers.noBody(), EMPTY_HASH); + try (InputStream ignored = response.body()) { + if (!successful(response)) throw failure(response); + observedOperations.add("DeleteObject"); + } + } + + /** Lists one page of objects using S3 ListObjectsV2. Pass the returned token to get the next page. */ + public ObjectPage listObjects(String bucket, String prefix, String continuationToken, int maxKeys) + throws IOException { + if (maxKeys < 1 || maxKeys > 1000) throw new IllegalArgumentException("maxKeys must be 1..1000"); + Map query = new HashMap<>(); + query.put("list-type", "2"); + query.put("encoding-type", "url"); + query.put("max-keys", Integer.toString(maxKeys)); + if (prefix != null) query.put("prefix", prefix); + if (continuationToken != null) query.put("continuation-token", continuationToken); + Element root = xml("GET", bucketPath(bucket), query); + requireRoot(root, "ListBucketResult"); + observedOperations.add("ListObjectsV2"); + List entries = new ArrayList<>(); + var nodes = Xml.descendants(root, "Contents"); + for (int i = 0; i < nodes.getLength(); i++) { + Element entry = (Element) nodes.item(i); + String key = Xml.text(entry, "Key"); + String size = Xml.text(entry, "Size"); + if (key == null || size == null) throw new ProtocolException("Incomplete object listing"); + try { + entries.add(new ObjectEntry(URLDecoder.decode(key.replace("+", "%2B"), StandardCharsets.UTF_8), + Long.parseLong(size), Xml.text(entry, "ETag"))); + } catch (IllegalArgumentException e) { throw new ProtocolException("Invalid object listing entry", e); } + } + String next = Xml.text(root, "NextContinuationToken"); + if ("true".equalsIgnoreCase(Xml.text(root, "IsTruncated")) && (next == null || next.isEmpty())) + throw new ProtocolException("Truncated listing has no continuation token"); + return new ObjectPage(entries, next); + } + + /** Starts a standard S3 multipart upload. Keep the returned upload ID to resume or abort later. */ + public MultipartUpload createMultipartUpload(String bucket, String key, String contentType) throws IOException { + String path = objectPath(bucket, key); + Map headers = contentType == null ? Map.of() : Map.of("content-type", contentType); + HttpResponse response = send("POST", path, Map.of("uploads", ""), headers, + HttpRequest.BodyPublishers.noBody(), EMPTY_HASH); + Element root = responseXml(response, "InitiateMultipartUploadResult"); + String id = Xml.text(root, "UploadId"); + if (id == null || id.isBlank()) throw new ProtocolException("Multipart initiation has no upload ID"); + observedOperations.add("CreateMultipartUpload"); + return new MultipartUpload(bucket, key, id); + } + + /** Uploads one part. The ETag in the result must be supplied when completing the upload. */ + public Part uploadPart(MultipartUpload upload, int partNumber, byte[] content) throws IOException { + Objects.requireNonNull(upload, "upload"); + Objects.requireNonNull(content, "content"); + checkPartNumber(partNumber); + HttpResponse response = send("PUT", objectPath(upload.bucket(), upload.key()), + Map.of("uploadId", upload.uploadId(), "partNumber", Integer.toString(partNumber)), Map.of(), + HttpRequest.BodyPublishers.ofByteArray(content), SigV4.hash(content)); + try (InputStream ignored = response.body()) { + if (!successful(response)) throw failure(response); + String etag = response.headers().firstValue("etag").orElse(null); + if (etag == null || etag.isBlank()) throw new ProtocolException("Part upload has no ETag"); + observedOperations.add("UploadPart"); + return new Part(partNumber, etag, content.length); + } + } + + /** + * Uploads an immutable file as sequential parts without completing it. The caller owns the + * upload session and can retry, list parts, complete, or abort it after any failure. + * Each part is held in memory once; partSize must be 5 to 128 MiB. + */ + public List uploadFileParts(MultipartUpload upload, Path file, int partSize) throws IOException { + Objects.requireNonNull(upload, "upload"); + Objects.requireNonNull(file, "file"); + if (partSize < 5 * 1024 * 1024 || partSize > 128 * 1024 * 1024) + throw new IllegalArgumentException("partSize must be 5..128 MiB"); + long size = Files.size(file); + if (size < 1 || (size + partSize - 1L) / partSize > 10000) + throw new IllegalArgumentException("file requires 1..10000 parts"); + List parts = new ArrayList<>(); + try (InputStream input = Files.newInputStream(file)) { + long remaining = size; + for (int number = 1; remaining > 0; number++) { + int length = (int) Math.min(remaining, partSize); + byte[] bytes = input.readNBytes(length); + if (bytes.length != length) throw new IOException("File changed during multipart upload"); + parts.add(uploadPart(upload, number, bytes)); + remaining -= length; + } + if (input.read() != -1) throw new IOException("File changed during multipart upload"); + } + return List.copyOf(parts); + } + + /** Lists one page of uploaded parts for resume or verification. */ + public PartPage listParts(MultipartUpload upload, int partNumberMarker, int maxParts) throws IOException { + Objects.requireNonNull(upload, "upload"); + if (partNumberMarker < 0 || partNumberMarker > 10000 || maxParts < 1 || maxParts > 1000) + throw new IllegalArgumentException("invalid part marker or page size"); + Element root = xml("GET", objectPath(upload.bucket(), upload.key()), + Map.of("uploadId", upload.uploadId(), "part-number-marker", Integer.toString(partNumberMarker), + "max-parts", Integer.toString(maxParts))); + requireRoot(root, "ListPartsResult"); + observedOperations.add("ListParts"); + List parts = new ArrayList<>(); + var nodes = Xml.descendants(root, "Part"); + for (int i = 0; i < nodes.getLength(); i++) { + Element entry = (Element) nodes.item(i); + try { + parts.add(new Part(Integer.parseInt(requiredText(entry, "PartNumber")), + requiredText(entry, "ETag"), Long.parseLong(requiredText(entry, "Size")))); + } catch (NumberFormatException e) { throw new ProtocolException("Invalid part listing entry", e); } + } + boolean truncated = Boolean.parseBoolean(Xml.text(root, "IsTruncated")); + int next = partNumberMarker; + if (truncated) { + try { next = Integer.parseInt(requiredText(root, "NextPartNumberMarker")); } + catch (NumberFormatException e) { throw new ProtocolException("Invalid next part marker", e); } + if (next <= partNumberMarker) throw new ProtocolException("Part listing did not advance"); + } + return new PartPage(parts, truncated, next); + } + + /** Completes the upload using the exact part numbers and ETags returned by the service. */ + public String completeMultipartUpload(MultipartUpload upload, List parts) throws IOException { + Objects.requireNonNull(upload, "upload"); + Objects.requireNonNull(parts, "parts"); + if (parts.isEmpty() || parts.size() > 10000) throw new IllegalArgumentException("1..10000 parts required"); + List ordered = new ArrayList<>(parts); + ordered.sort(Comparator.comparingInt(Part::number)); + StringBuilder xml = new StringBuilder(""); + int previous = 0; + for (Part part : ordered) { + Objects.requireNonNull(part, "part"); + checkPartNumber(part.number()); + if (part.number() == previous) throw new IllegalArgumentException("duplicate part number"); + previous = part.number(); + xml.append("").append(part.number()).append("") + .append(Xml.escape(part.etag())).append(""); + } + byte[] bytes = xml.append("").toString().getBytes(StandardCharsets.UTF_8); + HttpResponse response = send("POST", objectPath(upload.bucket(), upload.key()), + Map.of("uploadId", upload.uploadId()), Map.of("content-type", "application/xml"), + HttpRequest.BodyPublishers.ofByteArray(bytes), SigV4.hash(bytes)); + Element root = responseXml(response, "CompleteMultipartUploadResult"); + observedOperations.add("CompleteMultipartUpload"); + return requiredText(root, "ETag"); + } + + /** Aborts an unfinished upload. A completed upload cannot be aborted. */ + public void abortMultipartUpload(MultipartUpload upload) throws IOException { + Objects.requireNonNull(upload, "upload"); + HttpResponse response = send("DELETE", objectPath(upload.bucket(), upload.key()), + Map.of("uploadId", upload.uploadId()), Map.of(), HttpRequest.BodyPublishers.noBody(), EMPTY_HASH); + try (InputStream ignored = response.body()) { + if (!successful(response)) throw failure(response); + observedOperations.add("AbortMultipartUpload"); + } + } + + /** Lists one page of unfinished uploads in a bucket. */ + public MultipartUploadPage listMultipartUploads(String bucket, String prefix, String keyMarker, + String uploadIdMarker, int maxUploads) throws IOException { + if (maxUploads < 1 || maxUploads > 1000) throw new IllegalArgumentException("maxUploads must be 1..1000"); + if (uploadIdMarker != null && keyMarker == null) + throw new IllegalArgumentException("upload ID marker requires key marker"); + Map query = new HashMap<>(); + query.put("uploads", ""); + query.put("max-uploads", Integer.toString(maxUploads)); + if (prefix != null) query.put("prefix", prefix); + if (keyMarker != null) query.put("key-marker", keyMarker); + if (uploadIdMarker != null) query.put("upload-id-marker", uploadIdMarker); + Element root = xml("GET", bucketPath(bucket), query); + requireRoot(root, "ListMultipartUploadsResult"); + observedOperations.add("ListMultipartUploads"); + List uploads = new ArrayList<>(); + var nodes = Xml.descendants(root, "Upload"); + for (int i = 0; i < nodes.getLength(); i++) { + Element entry = (Element) nodes.item(i); + uploads.add(new MultipartUpload(bucket, requiredText(entry, "Key"), requiredText(entry, "UploadId"))); + } + boolean truncated = Boolean.parseBoolean(Xml.text(root, "IsTruncated")); + String nextKey = Xml.text(root, "NextKeyMarker"); + String nextId = Xml.text(root, "NextUploadIdMarker"); + if (truncated && (nextKey == null || nextId == null)) + throw new ProtocolException("Truncated upload listing has no markers"); + return new MultipartUploadPage(uploads, truncated, nextKey, nextId); + } + + private static String requiredText(Element element, String name) throws ProtocolException { + String value = Xml.text(element, name); + if (value == null || value.isBlank()) throw new ProtocolException("Multipart response missing " + name); + return value; + } + + private static void checkPartNumber(int number) { + if (number < 1 || number > 10000) throw new IllegalArgumentException("part number must be 1..10000"); + } + + /** Gets a bucket ACL. The backend may reject ACL operations when ACLs are disabled. */ + public AclPolicy getBucketAcl(String bucket) throws IOException { + AclPolicy policy = readAcl(bucketPath(bucket)); + observedOperations.add("GetBucketAcl"); + return policy; + } + + /** Gets an object ACL. This does not calculate permissions from policies or other grants. */ + public AclPolicy getObjectAcl(String bucket, String key) throws IOException { + AclPolicy policy = readAcl(objectPath(bucket, key)); + observedOperations.add("GetObjectAcl"); + return policy; + } + + /** Replaces a bucket ACL. No automatic retry is performed. */ + public void putBucketAcl(String bucket, AclPolicy policy) throws IOException { + writeAcl(bucketPath(bucket), policy); + observedOperations.add("PutBucketAcl"); + } + + /** Replaces an object ACL. No automatic retry is performed. */ + public void putObjectAcl(String bucket, String key, AclPolicy policy) throws IOException { + writeAcl(objectPath(bucket, key), policy); + observedOperations.add("PutObjectAcl"); + } + + private AclPolicy readAcl(String path) throws IOException { + Element root = xml("GET", path, Map.of("acl", "")); + requireRoot(root, "AccessControlPolicy"); + Element owner = Xml.child(root, "Owner"); + String ownerId = owner == null ? null : Xml.text(owner, "ID"); + if (ownerId == null || ownerId.isBlank()) throw new ProtocolException("ACL response has no owner ID"); + List grants = new ArrayList<>(); + var nodes = Xml.descendants(root, "Grant"); + for (int i = 0; i < nodes.getLength(); i++) { + Element grant = (Element) nodes.item(i); + Element grantee = Xml.child(grant, "Grantee"); + if (grantee == null) throw new ProtocolException("ACL grant has no grantee"); + String type = grantee.getAttributeNS("http://www.w3.org/2001/XMLSchema-instance", "type"); + if (type.isEmpty()) type = grantee.getAttribute("xsi:type"); + AclPolicy.GranteeType kind = switch (type) { + case "CanonicalUser" -> AclPolicy.GranteeType.CANONICAL_USER; + case "Group" -> AclPolicy.GranteeType.GROUP; + default -> throw new ProtocolException("Unsupported ACL grantee type"); + }; + String id = Xml.text(grantee, kind == AclPolicy.GranteeType.GROUP ? "URI" : "ID"); + String permission = Xml.text(grant, "Permission"); + if (id == null || permission == null) throw new ProtocolException("Incomplete ACL grant"); + try { grants.add(new AclPolicy.Grant(kind, id, AclPolicy.Permission.valueOf(permission))); } + catch (IllegalArgumentException e) { throw new ProtocolException("Unsupported ACL permission", e); } + } + return new AclPolicy(ownerId, grants); + } + + private void writeAcl(String path, AclPolicy policy) throws IOException { + Objects.requireNonNull(policy, "policy"); + StringBuilder xml = new StringBuilder("") + .append(Xml.escape(policy.ownerId())).append(""); + for (AclPolicy.Grant grant : policy.grants()) { + boolean group = grant.type() == AclPolicy.GranteeType.GROUP; + xml.append("<").append(group ? "URI" : "ID").append('>') + .append(Xml.escape(grant.grantee())).append("").append(grant.permission()).append(""); + } + xml.append(""); + byte[] bytes = xml.toString().getBytes(StandardCharsets.UTF_8); + String md5; + try { md5 = Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(bytes)); } + catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); } + HttpResponse response = send("PUT", path, Map.of("acl", ""), + Map.of("content-type", "application/xml", "content-md5", md5), + HttpRequest.BodyPublishers.ofByteArray(bytes), SigV4.hash(bytes)); + try (InputStream ignored = response.body()) { + if (!successful(response)) throw failure(response); + } + } + + /** + * Reads ObjectStore's signed manifest when present. Other S3 endpoints report only operations + * observed succeeding on this client. A denied or missing manifest does not imply a feature is + * unsupported. Service identity from /health is self-reported, not cryptographic attestation. + */ + public Capabilities getCapabilities() throws IOException { + HttpResponse response = send("GET", "/_objectstore/capabilities", Map.of(), Map.of(), + HttpRequest.BodyPublishers.noBody(), EMPTY_HASH); + ProtocolException invalidManifest = null; + try (InputStream body = response.body()) { + if (successful(response)) { + try { return withObserved(parseManifest(readLimited(body, 64 * 1024))); } + catch (ProtocolException e) { invalidManifest = e; } + } + } + boolean objectStore = probeHealth(); + if (invalidManifest != null && objectStore) throw invalidManifest; + return withObserved(new Capabilities(objectStore ? Capabilities.ServiceKind.OBJECTSTORE : + Capabilities.ServiceKind.UNKNOWN_S3, Map.of())); + } + + /** + * Opts into read-only probes on a known bucket and, optionally, an existing object key. + * Successful calls establish support for this principal. S3 error responses leave support + * UNKNOWN; transport and malformed-response errors are still reported to the caller. + */ + public Capabilities probeReadOnlyCapabilities(String bucket, String existingObjectKey) throws IOException { + Capabilities base = getCapabilities(); + probe(base, "ListObjectsV2", () -> listObjects(bucket, null, null, 1)); + probe(base, "ListMultipartUploads", () -> listMultipartUploads(bucket, null, null, null, 1)); + probe(base, "GetBucketAcl", () -> getBucketAcl(bucket)); + if (existingObjectKey != null) { + probe(base, "HeadObject", () -> headObject(bucket, existingObjectKey)); + probe(base, "GetObjectAcl", () -> getObjectAcl(bucket, existingObjectKey)); + } + return withObserved(base); + } + + private void probe(Capabilities base, String operation, Probe call) throws IOException { + if (base.support(operation) == Capabilities.Support.UNSUPPORTED) return; + try { call.run(); } + catch (ObjectStorageException ignored) { } + } + + @FunctionalInterface private interface Probe { void run() throws IOException; } + + private Capabilities withObserved(Capabilities base) { + Map operations = new HashMap<>(base.operations()); + observedOperations.forEach(name -> operations.put(name, Capabilities.Support.SUPPORTED)); + return new Capabilities(base.service(), operations, base.limits(), base.serviceVersion(), + base.storageMode(), base.completeOperationInventory()); + } + + private static Capabilities parseManifest(byte[] bytes) throws ProtocolException { + Map document = Json.object(Json.parse(bytes), "capability manifest"); + if (Json.integer(document.get("schemaVersion"), "schemaVersion") != 1 || + !"lunarsky-objectstore".equals(Json.string(document.get("service"), "service"))) + throw new ProtocolException("Unsupported capability manifest"); + String version = Json.string(document.get("serviceVersion"), "serviceVersion"); + String mode = Json.string(document.get("storageMode"), "storageMode"); + Map operations = new HashMap<>(); + for (Object value : Json.array(document.get("operations"), "operations")) { + String name = Json.string(value, "operation name"); + if (operations.put(name, Capabilities.Support.SUPPORTED) != null) + throw new ProtocolException("Duplicate capability operation"); + } + Map rawLimits = Json.object(document.get("limits"), "limits"); + Map limits = new HashMap<>(); + for (var entry : rawLimits.entrySet()) { + long value = Json.integer(entry.getValue(), "limit " + entry.getKey()); + if (value < 0) throw new ProtocolException("Negative capability limit"); + limits.put(entry.getKey(), value); + } + for (String required : List.of("maxObjectBytes", "maxTotalBytes", "maxParts")) + if (!limits.containsKey(required)) throw new ProtocolException("Missing capability limit " + required); + return new Capabilities(Capabilities.ServiceKind.OBJECTSTORE, operations, limits, version, mode, true); + } + + private boolean probeHealth() throws IOException { + URI uri = URI.create(origin() + "/health"); + HttpRequest request = HttpRequest.newBuilder(uri).timeout(timeout).GET().build(); + HttpResponse response = execute(request); + try (InputStream body = response.body()) { + if (response.statusCode() != 200) return false; + String value = new String(readLimited(body, 1024), StandardCharsets.UTF_8); + return value.matches("(?s)\\s*\\{\\s*\"status\"\\s*:\\s*\"ok\"\\s*,\\s*\"service\"\\s*:\\s*\"lunarsky-objectstore\"\\s*}\\s*"); + } + } + + private Element xml(String method, String path, Map query) throws IOException { + HttpResponse response = send(method, path, query, Map.of(), + HttpRequest.BodyPublishers.noBody(), EMPTY_HASH); + return responseXml(response, null); + } + + private static Element responseXml(HttpResponse response, String expectedRoot) throws IOException { + try (InputStream body = response.body()) { + if (!successful(response)) throw failure(response); + Element root = Xml.parse(readLimited(body, MAX_XML)).getDocumentElement(); + if ("Error".equals(root.getLocalName())) { + String code = Xml.text(root, "Code"); + throw new ObjectStorageException(response.statusCode(), code, + response.headers().firstValue("x-amz-request-id").orElse(null)); + } + if (expectedRoot != null) requireRoot(root, expectedRoot); + return root; + } + } + + private HttpResponse send(String method, String path, Map query, + Map extra, HttpRequest.BodyPublisher body, + String payloadHash) throws IOException { + String queryString = SigV4.query(query); + URI uri = URI.create(origin() + path + (queryString.isEmpty() ? "" : "?" + queryString)); + Instant now = clock.instant(); + Map headers = new HashMap<>(extra); + headers.put("host", uri.getRawAuthority().toLowerCase(Locale.ROOT)); + headers.put("x-amz-date", SigV4.timestamp(now)); + headers.put("x-amz-content-sha256", payloadHash); + String authorization = SigV4.authorization(method, uri, headers, payloadHash, now, + region, accessKey, secretKey); + HttpRequest.Builder request = HttpRequest.newBuilder(uri).timeout(timeout); + headers.forEach((name, value) -> { + if (!"host".equals(name)) request.header(name, value); + }); + request.header("Authorization", authorization); + return execute(request.method(method, body).build()); + } + + private HttpResponse execute(HttpRequest request) throws IOException { + try { return http.send(request, HttpResponse.BodyHandlers.ofInputStream()); } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Storage request interrupted", e); + } catch (IOException e) { throw new TransportException(e); } + } + + private static boolean successful(HttpResponse response) { + return response.statusCode() >= 200 && response.statusCode() < 300; + } + + private static ObjectStorageException failure(HttpResponse response) throws IOException { + String code = null; + String requestId = response.headers().firstValue("x-amz-request-id").orElse(null); + try { + byte[] bytes = readLimited(response.body(), MAX_ERROR); + if (bytes.length > 0) { + Element root = Xml.parse(bytes).getDocumentElement(); + if ("Error".equals(root.getLocalName())) { + code = Xml.text(root, "Code"); + if (requestId == null) requestId = Xml.text(root, "RequestId"); + } + } + } catch (IOException ignored) { + } + return new ObjectStorageException(response.statusCode(), code, requestId); + } + + private static byte[] readLimited(InputStream stream, int limit) throws IOException { + byte[] bytes = stream.readNBytes(limit + 1); + if (bytes.length > limit) throw new ProtocolException("Storage response exceeds size limit"); + return bytes; + } + + private static long length(HttpResponse response) { + return response.headers().firstValueAsLong("content-length").orElse(-1); + } + + private static void requireRoot(Element root, String name) throws IOException { + if (!name.equals(root.getLocalName())) throw new ProtocolException("Unexpected storage XML response"); + } + + private String origin() { + return endpoint.getScheme() + "://" + endpoint.getRawAuthority(); + } + + private static String bucketPath(String bucket) { + validateBucket(bucket); + return "/" + SigV4.encode(bucket, false); + } + + private static String objectPath(String bucket, String key) { + if (key == null || key.isEmpty() || key.indexOf('\0') >= 0 || + key.getBytes(StandardCharsets.UTF_8).length > 1024) + throw new IllegalArgumentException("object key must be 1..1024 UTF-8 bytes without NUL"); + return bucketPath(bucket) + "/" + SigV4.encode(key, true); + } + + private static void validateBucket(String bucket) { + if (bucket == null || !bucket.matches("[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]")) + throw new IllegalArgumentException("bucket must be a 3..63 character DNS-style name"); + } + + /** Closes the underlying HTTP client. Do not use this instance afterward. */ + @Override public void close() { http.close(); } + + /** An open response stream and selected object headers. Close this result after reading. */ + public record ObjectData(InputStream body, long length, String contentType, String etag) + implements AutoCloseable { + @Override public void close() throws IOException { body.close(); } + } + + /** Headers returned by a HEAD request. A negative length means it was not supplied. */ + public record ObjectMetadata(long length, String contentType, String etag) { } + + /** A listed object. */ + public record ObjectEntry(String key, long size, String etag) { } + + /** One listing page. A null next token means no next page was reported. */ + public record ObjectPage(List objects, String nextContinuationToken) { + public ObjectPage { objects = List.copyOf(objects); } + } + + /** An unfinished multipart upload. Persist all three fields if resuming in a later process. */ + public record MultipartUpload(String bucket, String key, String uploadId) { + public MultipartUpload { + validateBucket(bucket); + if (key == null || key.isEmpty()) throw new IllegalArgumentException("key is required"); + if (uploadId == null || uploadId.isBlank()) throw new IllegalArgumentException("upload ID is required"); + } + } + + /** A successfully uploaded part, including the service-provided ETag. */ + public record Part(int number, String etag, long size) { + public Part { + checkPartNumber(number); + if (etag == null || etag.isBlank()) throw new IllegalArgumentException("ETag is required"); + if (size < 0) throw new IllegalArgumentException("part size must not be negative"); + } + } + + /** A page of uploaded parts. Pass nextPartNumberMarker to the next call when truncated. */ + public record PartPage(List parts, boolean truncated, int nextPartNumberMarker) { + public PartPage { parts = List.copyOf(parts); } + } + + /** A page of unfinished uploads. Pass both next markers to the next call when truncated. */ + public record MultipartUploadPage(List uploads, boolean truncated, + String nextKeyMarker, String nextUploadIdMarker) { + public MultipartUploadPage { uploads = List.copyOf(uploads); } + } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClientBuilder.java b/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClientBuilder.java new file mode 100644 index 0000000..3792c12 --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageClientBuilder.java @@ -0,0 +1,70 @@ +package cloud.lunarsky.objectstore.client; + +import java.net.URI; +import java.time.Clock; +import java.time.Duration; +import java.util.Objects; + +/** Configures a path-style, Signature V4 S3 client without third-party dependencies. */ +public final class ObjectStorageClientBuilder { + private URI endpoint; + private String region = "us-east-1"; + private String accessKey; + private String secretKey; + private Duration timeout = Duration.ofSeconds(30); + private boolean allowInsecureHttp; + + /** Sets the service root, for example {@code https://storage.example.com}. */ + public ObjectStorageClientBuilder endpoint(URI value) { + endpoint = Objects.requireNonNull(value, "endpoint"); + return this; + } + + /** Sets the Signature V4 region. The default is {@code us-east-1}. */ + public ObjectStorageClientBuilder region(String value) { + region = requireText(value, "region"); + return this; + } + + /** Sets credentials used only in request signatures. Keep them out of logs. */ + public ObjectStorageClientBuilder credentials(String access, String secret) { + accessKey = requireText(access, "access key"); + secretKey = requireText(secret, "secret key"); + return this; + } + + /** Sets the per-request timeout. */ + public ObjectStorageClientBuilder timeout(Duration value) { + if (Objects.requireNonNull(value, "timeout").isNegative() || value.isZero()) + throw new IllegalArgumentException("timeout must be positive"); + timeout = value; + return this; + } + + /** Allows plain HTTP for a trusted local test endpoint. HTTPS is required by default. */ + public ObjectStorageClientBuilder allowInsecureHttp() { + allowInsecureHttp = true; + return this; + } + + /** Builds an independent client. */ + public ObjectStorageClient build() { + if (endpoint == null) throw new IllegalStateException("endpoint is required"); + if (accessKey == null || secretKey == null) throw new IllegalStateException("credentials are required"); + if (!region.matches("[a-z0-9-]+")) throw new IllegalArgumentException("invalid region"); + if (!accessKey.matches("[A-Za-z0-9_+=./@-]+")) throw new IllegalArgumentException("invalid access key"); + String scheme = endpoint.getScheme(); + if (!"https".equalsIgnoreCase(scheme) && !(allowInsecureHttp && "http".equalsIgnoreCase(scheme))) + throw new IllegalArgumentException("HTTPS endpoint required unless insecure HTTP is explicitly allowed"); + if (endpoint.getHost() == null || endpoint.getUserInfo() != null || endpoint.getRawQuery() != null || + endpoint.getRawFragment() != null || !(endpoint.getRawPath() == null || endpoint.getRawPath().isEmpty() || + "/".equals(endpoint.getRawPath()))) + throw new IllegalArgumentException("endpoint must be an origin without path, query, fragment, or user info"); + return new ObjectStorageClient(endpoint, region, accessKey, secretKey, timeout, Clock.systemUTC()); + } + + private static String requireText(String value, String name) { + if (value == null || value.isBlank()) throw new IllegalArgumentException(name + " is required"); + return value; + } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageException.java b/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageException.java new file mode 100644 index 0000000..8143f85 --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/ObjectStorageException.java @@ -0,0 +1,30 @@ +package cloud.lunarsky.objectstore.client; + +import java.io.IOException; + +/** An S3 error response with stable fields for callers to inspect. */ +public final class ObjectStorageException extends IOException { + private final int statusCode; + private final String errorCode; + private final String requestId; + + ObjectStorageException(int statusCode, String errorCode, String requestId) { + super("Storage request failed: HTTP " + statusCode + (errorCode == null ? "" : " (" + errorCode + ")")); + this.statusCode = statusCode; + this.errorCode = errorCode; + this.requestId = requestId; + } + + /** Returns the HTTP status. */ + public int statusCode() { return statusCode; } + + /** Returns the S3 error code, or {@code null} if the server supplied none. */ + public String errorCode() { return errorCode; } + + /** Returns the request ID, or {@code null}. */ + public String requestId() { return requestId; } + + /** Returns whether retry might help. A failed write may already have reached the server. */ + public boolean retryable() { return statusCode == 429 || statusCode == 500 || statusCode == 502 || + statusCode == 503 || statusCode == 504; } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/ProtocolException.java b/client/src/main/java/cloud/lunarsky/objectstore/client/ProtocolException.java new file mode 100644 index 0000000..4affc2d --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/ProtocolException.java @@ -0,0 +1,9 @@ +package cloud.lunarsky.objectstore.client; + +import java.io.IOException; + +/** A successful HTTP response that does not match the expected storage protocol. */ +public final class ProtocolException extends IOException { + ProtocolException(String message) { super(message); } + ProtocolException(String message, Throwable cause) { super(message, cause); } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/SigV4.java b/client/src/main/java/cloud/lunarsky/objectstore/client/SigV4.java new file mode 100644 index 0000000..46f1cb0 --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/SigV4.java @@ -0,0 +1,83 @@ +package cloud.lunarsky.objectstore.client; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +final class SigV4 { + private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.ROOT) + .withZone(ZoneOffset.UTC); + private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'", Locale.ROOT) + .withZone(ZoneOffset.UTC); + + private SigV4() { } + + static String hash(byte[] bytes) { + try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); } + catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); } + } + + static String encode(String text, boolean keepSlash) { + StringBuilder result = new StringBuilder(); + for (byte value : text.getBytes(StandardCharsets.UTF_8)) { + int b = value & 0xff; + if (b >= 'A' && b <= 'Z' || b >= 'a' && b <= 'z' || b >= '0' && b <= '9' || + b == '-' || b == '_' || b == '.' || b == '~' || keepSlash && b == '/') result.append((char) b); + else result.append('%').append(Character.toUpperCase(Character.forDigit(b >>> 4, 16))) + .append(Character.toUpperCase(Character.forDigit(b & 15, 16))); + } + return result.toString(); + } + + static String query(Map values) { + List entries = new ArrayList<>(); + values.forEach((key, value) -> entries.add(encode(key, false) + "=" + encode(value, false))); + entries.sort(String::compareTo); + return String.join("&", entries); + } + + static String authorization(String method, URI uri, Map headers, String payloadHash, + Instant now, String region, String accessKey, String secretKey) { + TreeMap signed = new TreeMap<>(); + headers.forEach((name, value) -> signed.put(name.toLowerCase(Locale.ROOT), + value.trim().replaceAll("\\s+", " "))); + String names = String.join(";", signed.keySet()); + StringBuilder canonicalHeaders = new StringBuilder(); + signed.forEach((name, value) -> canonicalHeaders.append(name).append(':').append(value).append('\n')); + String canonical = method + '\n' + uri.getRawPath() + '\n' + + (uri.getRawQuery() == null ? "" : uri.getRawQuery()) + '\n' + canonicalHeaders + '\n' + + names + '\n' + payloadHash; + String day = DATE.format(now); + String scope = day + '/' + region + "/s3/aws4_request"; + String toSign = "AWS4-HMAC-SHA256\n" + TIME.format(now) + '\n' + scope + '\n' + + hash(canonical.getBytes(StandardCharsets.UTF_8)); + byte[] key = hmac(("AWS4" + secretKey).getBytes(StandardCharsets.UTF_8), day); + key = hmac(key, region); + key = hmac(key, "s3"); + key = hmac(key, "aws4_request"); + return "AWS4-HMAC-SHA256 Credential=" + accessKey + '/' + scope + ",SignedHeaders=" + names + + ",Signature=" + HexFormat.of().formatHex(hmac(key, toSign)); + } + + private static byte[] hmac(byte[] key, String value) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(value.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { throw new IllegalStateException("HMAC-SHA256 unavailable", e); } + } + + static String timestamp(Instant now) { return TIME.format(now); } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/TransportException.java b/client/src/main/java/cloud/lunarsky/objectstore/client/TransportException.java new file mode 100644 index 0000000..9fcf330 --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/TransportException.java @@ -0,0 +1,8 @@ +package cloud.lunarsky.objectstore.client; + +import java.io.IOException; + +/** A connection or I/O failure. A write may have completed before this was observed. */ +public final class TransportException extends IOException { + TransportException(IOException cause) { super("Storage transport failed", cause); } +} diff --git a/client/src/main/java/cloud/lunarsky/objectstore/client/Xml.java b/client/src/main/java/cloud/lunarsky/objectstore/client/Xml.java new file mode 100644 index 0000000..0b128cd --- /dev/null +++ b/client/src/main/java/cloud/lunarsky/objectstore/client/Xml.java @@ -0,0 +1,54 @@ +package cloud.lunarsky.objectstore.client; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +final class Xml { + private Xml() { } + + static Document parse(byte[] bytes) throws IOException { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + return factory.newDocumentBuilder().parse(new ByteArrayInputStream(bytes)); + } catch (ParserConfigurationException | SAXException | IllegalArgumentException e) { + throw new ProtocolException("Malformed or unsafe XML response", e); + } + } + + static Element child(Element element, String name) { + for (Node node = element.getFirstChild(); node != null; node = node.getNextSibling()) + if (node instanceof Element found && name.equals(found.getLocalName())) return found; + return null; + } + + static String text(Element element, String name) { + Element found = child(element, name); + return found == null ? null : found.getTextContent(); + } + + static NodeList descendants(Element element, String name) { + return element.getElementsByTagNameNS("*", name); + } + + static String escape(String text) { + return text.replace("&", "&").replace("<", "<").replace(">", ">") + .replace("\"", """).replace("'", "'"); + } +} diff --git a/client/src/test/java/cloud/lunarsky/objectstore/client/ClientTest.java b/client/src/test/java/cloud/lunarsky/objectstore/client/ClientTest.java new file mode 100644 index 0000000..bd00754 --- /dev/null +++ b/client/src/test/java/cloud/lunarsky/objectstore/client/ClientTest.java @@ -0,0 +1,190 @@ +package cloud.lunarsky.objectstore.client; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public final class ClientTest { + private static final String S3_NS = "http://s3.amazonaws.com/doc/2006-03-01/"; + + public static void main(String[] args) throws Exception { + signingVector(); + protocol(); + System.out.println("Client tests passed"); + } + + private static void signingVector() { + URI uri = URI.create("https://examplebucket.s3.amazonaws.com/test.txt"); + String empty = SigV4.hash(new byte[0]); + String actual = SigV4.authorization("GET", uri, Map.of( + "host", "examplebucket.s3.amazonaws.com", "range", "bytes=0-9", + "x-amz-content-sha256", empty, "x-amz-date", "20130524T000000Z"), empty, + Instant.parse("2013-05-24T00:00:00Z"), "us-east-1", "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + check(actual.endsWith("Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"), + "AWS Signature V4 vector"); + check("a=%20&z=%2F".equals(SigV4.query(Map.of("z", "/", "a", " "))), "query encoding"); + } + + private static void protocol() throws Exception { + AtomicReference observedHash = new AtomicReference<>(); + AtomicReference observedAcl = new AtomicReference<>(); + AtomicReference manifest = new AtomicReference<>(); + AtomicInteger manifestStatus = new AtomicInteger(200); + AtomicBoolean health = new AtomicBoolean(true); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try { handle(exchange, observedHash, observedAcl, manifest, manifestStatus, health); } + finally { exchange.close(); } + }); + server.start(); + try (ObjectStorageClient client = new ObjectStorageClientBuilder() + .endpoint(URI.create("http://127.0.0.1:" + server.getAddress().getPort())) + .allowInsecureHttp().credentials("test-key", "test-secret").build()) { + client.putObject("mybucket", "hello world.txt", "hello".getBytes(StandardCharsets.UTF_8), "text/plain"); + check(SigV4.hash("hello".getBytes(StandardCharsets.UTF_8)).equals(observedHash.get()), "payload hash"); + try (ObjectStorageClient.ObjectData data = client.getObject("mybucket", "hello world.txt")) { + check("hello".equals(new String(data.body().readAllBytes(), StandardCharsets.UTF_8)), "GET body"); + } + check("text/plain".equals(client.headObject("mybucket", "hello world.txt").contentType()), + "HEAD content type"); + var page = client.listObjects("mybucket", "hello", null, 10); + check(page.objects().size() == 1 && "hello world.txt".equals(page.objects().getFirst().key()), + "list object key"); + check("next-token".equals(page.nextContinuationToken()), "list continuation"); + AclPolicy policy = client.getBucketAcl("mybucket"); + check(policy.grants().size() == 1 && policy.grants().getFirst().permission() == + AclPolicy.Permission.FULL_CONTROL, "GET ACL"); + client.putObjectAcl("mybucket", "hello world.txt", new AclPolicy("owner-id", List.of( + new AclPolicy.Grant(AclPolicy.GranteeType.GROUP, + "http://acs.amazonaws.com/groups/global/AllUsers", AclPolicy.Permission.READ)))); + check(observedAcl.get().contains("READ"), "PUT ACL"); + check(client.getCapabilities().service() == Capabilities.ServiceKind.OBJECTSTORE, "service discovery"); + check(client.getCapabilities().support("versioning") == Capabilities.Support.UNKNOWN, + "unknown support remains unknown"); + manifest.set("{\"schemaVersion\":1,\"service\":\"lunarsky-objectstore\"," + + "\"serviceVersion\":\"0.0.5\",\"storageMode\":\"disk\"," + + "\"operations\":[\"PutObject\",\"ListParts\"]," + + "\"limits\":{\"maxObjectBytes\":1024,\"maxTotalBytes\":4096,\"maxParts\":10000}}"); + var capabilities = client.getCapabilities(); + check(capabilities.service() == Capabilities.ServiceKind.OBJECTSTORE && + capabilities.support("ListParts") == Capabilities.Support.SUPPORTED && + capabilities.support("PutBucketAcl") == Capabilities.Support.UNSUPPORTED && + capabilities.limits().get("maxParts") == 10000 && + "0.0.5".equals(capabilities.serviceVersion()), "signed manifest"); + manifest.set("{\"schemaVersion\":1,\"schemaVersion\":1}"); + try { + client.getCapabilities(); + throw new AssertionError("Expected malformed manifest"); + } catch (ProtocolException expected) { } + health.set(false); + check(client.getCapabilities().service() == Capabilities.ServiceKind.UNKNOWN_S3, + "generic S3 with unrelated matching path"); + manifestStatus.set(503); + check(client.getCapabilities().support("CopyObject") == Capabilities.Support.UNKNOWN, + "ambiguous manifest failure stays unknown"); + manifestStatus.set(200); + manifest.set(null); + check(client.getCapabilities().service() == Capabilities.ServiceKind.UNKNOWN_S3, "generic S3 fallback"); + check(client.getCapabilities().support("PutObject") == Capabilities.Support.SUPPORTED && + client.getCapabilities().support("CopyObject") == Capabilities.Support.UNKNOWN, + "generic S3 observed support"); + capabilities = client.probeReadOnlyCapabilities("mybucket", "hello world.txt"); + check(capabilities.support("ListMultipartUploads") == Capabilities.Support.UNKNOWN && + capabilities.support("ListObjectsV2") == Capabilities.Support.SUPPORTED, + "denied generic probe remains unknown"); + client.deleteObject("mybucket", "hello world.txt"); + try { + client.getObject("mybucket", "denied"); + throw new AssertionError("Expected AccessDenied"); + } catch (ObjectStorageException error) { + check(error.statusCode() == 403 && "AccessDenied".equals(error.errorCode()) && + "request-123".equals(error.requestId()), "typed S3 error"); + } + } finally { server.stop(0); } + } + + private static void handle(HttpExchange exchange, AtomicReference hash, + AtomicReference acl, AtomicReference manifest, + AtomicInteger manifestStatus, AtomicBoolean health) throws IOException { + String path = exchange.getRequestURI().getRawPath(); + String query = exchange.getRequestURI().getRawQuery(); + if ("/health".equals(path)) { + respond(exchange, health.get() ? 200 : 404, + health.get() ? "{\"status\":\"ok\",\"service\":\"lunarsky-objectstore\"}" : ""); + return; + } + check(exchange.getRequestHeaders().getFirst("Authorization") != null, "signed request"); + if ("/_objectstore/capabilities".equals(path)) { + String value = manifest.get(); + respond(exchange, value == null ? 404 : manifestStatus.get(), value == null ? "" : value); + return; + } + if ("/mybucket".equals(path) && query != null && query.contains("uploads=")) { + respond(exchange, 403, "AccessDenied"); + return; + } + if ("/mybucket".equals(path) && query != null && query.contains("list-type=2")) { + respond(exchange, 200, "hello%20world.txt" + + "5\"abc\"next-token" + + ""); + return; + } + if (query != null && query.equals("acl=")) { + if ("GET".equals(exchange.getRequestMethod())) { + respond(exchange, 200, "owner-id" + + "owner-id" + + "FULL_CONTROL"); + } else { + acl.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + check(exchange.getRequestHeaders().getFirst("Content-MD5") != null, "ACL Content-MD5"); + respond(exchange, 200, ""); + } + return; + } + if ("/mybucket/denied".equals(path)) { + exchange.getResponseHeaders().add("x-amz-request-id", "request-123"); + respond(exchange, 403, "AccessDenied"); + return; + } + check("/mybucket/hello%20world.txt".equals(path), "path-style key encoding"); + switch (exchange.getRequestMethod()) { + case "PUT" -> { + byte[] body = exchange.getRequestBody().readAllBytes(); + hash.set(exchange.getRequestHeaders().getFirst("x-amz-content-sha256")); + check(SigV4.hash(body).equals(hash.get()), "uploaded body hash"); + respond(exchange, 200, ""); + } + case "GET" -> respond(exchange, 200, "hello"); + case "HEAD" -> { + exchange.getResponseHeaders().add("Content-Type", "text/plain"); + exchange.sendResponseHeaders(200, -1); + } + case "DELETE" -> respond(exchange, 204, ""); + default -> throw new AssertionError("Unexpected method"); + } + } + + private static void respond(HttpExchange exchange, int status, String content) throws IOException { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + if (status == 204 || bytes.length == 0) exchange.sendResponseHeaders(status, -1); + else { + exchange.sendResponseHeaders(status, bytes.length); + exchange.getResponseBody().write(bytes); + } + } + + private static void check(boolean condition, String description) { + if (!condition) throw new AssertionError(description); + } +} diff --git a/client/src/test/java/cloud/lunarsky/objectstore/client/MultipartClientTest.java b/client/src/test/java/cloud/lunarsky/objectstore/client/MultipartClientTest.java new file mode 100644 index 0000000..711369b --- /dev/null +++ b/client/src/test/java/cloud/lunarsky/objectstore/client/MultipartClientTest.java @@ -0,0 +1,119 @@ +package cloud.lunarsky.objectstore.client; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public final class MultipartClientTest { + public static void main(String[] args) throws Exception { + AtomicReference completion = new AtomicReference<>(); + AtomicBoolean completionError = new AtomicBoolean(); + List sizes = new ArrayList<>(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try { handle(exchange, completion, completionError, sizes); } + finally { exchange.close(); } + }); + server.start(); + try (ObjectStorageClient client = new ObjectStorageClientBuilder() + .endpoint(URI.create("http://127.0.0.1:" + server.getAddress().getPort())) + .allowInsecureHttp().credentials("test-key", "test-secret").build()) { + var upload = client.createMultipartUpload("mybucket", "large file.bin", "application/octet-stream"); + check("u-1".equals(upload.uploadId()), "initiation ID"); + var first = client.uploadPart(upload, 1, "first".getBytes(StandardCharsets.UTF_8)); + var second = client.uploadPart(upload, 2, "second".getBytes(StandardCharsets.UTF_8)); + check(first.number() == 1 && "\"etag-1\"".equals(first.etag()), "part result"); + var page = client.listParts(upload, 0, 1); + check(page.truncated() && page.nextPartNumberMarker() == 1 && page.parts().size() == 1, + "first part page"); + page = client.listParts(upload, page.nextPartNumberMarker(), 1); + check(!page.truncated() && page.parts().getFirst().number() == 2, "second part page"); + var uploads = client.listMultipartUploads("mybucket", "large", null, null, 10); + check(uploads.uploads().size() == 1 && "u-1".equals(uploads.uploads().getFirst().uploadId()), + "unfinished upload listing"); + String etag = client.completeMultipartUpload(upload, List.of(second, first)); + check("\"final-etag\"".equals(etag), "completion ETag"); + check(completion.get().indexOf("1") < + completion.get().indexOf("2"), "completion part order"); + completionError.set(true); + try { + client.completeMultipartUpload(upload, List.of(first)); + throw new AssertionError("Expected embedded S3 error"); + } catch (ObjectStorageException error) { + check("InvalidPart".equals(error.errorCode()), "HTTP 200 completion error"); + } + completionError.set(false); + Path file = Files.createTempFile("objectstore-client-multipart", ".bin"); + try { + byte[] content = new byte[5 * 1024 * 1024 + 3]; + content[content.length - 1] = 42; + Files.write(file, content); + var fileParts = client.uploadFileParts(upload, file, 5 * 1024 * 1024); + check(fileParts.size() == 2 && fileParts.getLast().size() == 3, "file part slicing"); + check(sizes.contains(5 * 1024 * 1024) && sizes.contains(3), "part request sizes"); + } finally { Files.deleteIfExists(file); } + client.abortMultipartUpload(upload); + System.out.println("Multipart client tests passed"); + } finally { server.stop(0); } + } + + private static void handle(HttpExchange exchange, AtomicReference completion, + AtomicBoolean completionError, List sizes) throws IOException { + check(exchange.getRequestHeaders().getFirst("Authorization") != null, "signed multipart request"); + String path = exchange.getRequestURI().getRawPath(); + String query = exchange.getRequestURI().getRawQuery(); + String method = exchange.getRequestMethod(); + if ("/mybucket".equals(path) && "GET".equals(method) && query.contains("uploads=")) { + reply(exchange, 200, "large file.bin" + + "u-1false"); + return; + } + check("/mybucket/large%20file.bin".equals(path), "multipart path"); + if ("POST".equals(method) && "uploads=".equals(query)) { + reply(exchange, 200, "u-1" + + ""); + } else if ("PUT".equals(method) && query.contains("partNumber=")) { + int number = query.contains("partNumber=1") ? 1 : 2; + byte[] body = exchange.getRequestBody().readAllBytes(); + check(SigV4.hash(body).equals(exchange.getRequestHeaders().getFirst("x-amz-content-sha256")), + "part hash"); + sizes.add(body.length); + exchange.getResponseHeaders().set("ETag", "\"etag-" + number + "\""); + reply(exchange, 200, ""); + } else if ("GET".equals(method) && query.contains("uploadId=")) { + boolean first = query.contains("part-number-marker=0"); + reply(exchange, 200, "" + (first ? 1 : 2) + + "\"etag-" + (first ? 1 : 2) + "\"" + + (first ? 5 : 6) + "" + first + "" + + "" + (first ? 1 : 2) + ""); + } else if ("POST".equals(method) && query.contains("uploadId=")) { + completion.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + reply(exchange, 200, completionError.get() ? "InvalidPart" : + "\"final-etag\""); + } else if ("DELETE".equals(method) && query.contains("uploadId=")) { + reply(exchange, 204, ""); + } else throw new AssertionError("Unexpected multipart operation: " + method + " " + query); + } + + private static void reply(HttpExchange exchange, int status, String content) throws IOException { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + if (bytes.length == 0) exchange.sendResponseHeaders(status, -1); + else { + exchange.sendResponseHeaders(status, bytes.length); + exchange.getResponseBody().write(bytes); + } + } + + private static void check(boolean condition, String description) { + if (!condition) throw new AssertionError(description); + } +} diff --git a/compose.cluster.yaml b/compose.cluster.yaml index 96a6ab8..ecf066e 100644 --- a/compose.cluster.yaml +++ b/compose.cluster.yaml @@ -12,8 +12,15 @@ services: S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} S3_BUCKET: ${S3_BUCKET:-objects} S3_REGION: ${S3_REGION:-us-east-1} + S3_CREDENTIALS_FILE: ${S3_CREDENTIALS_FILE:-} MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728} MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648} + PUBLIC_REQUESTS_PER_SECOND: ${PUBLIC_REQUESTS_PER_SECOND:-0} + PUBLIC_REQUEST_BURST: ${PUBLIC_REQUEST_BURST:-} + PUBLIC_BYTES_PER_SECOND: ${PUBLIC_BYTES_PER_SECOND:-0} + PUBLIC_BYTE_BURST: ${PUBLIC_BYTE_BURST:-} + PUBLIC_MAX_IN_FLIGHT_PER_IP: ${PUBLIC_MAX_IN_FLIGHT_PER_IP:-} + PUBLIC_TRUSTED_PROXY_IPS: ${PUBLIC_TRUSTED_PROXY_IPS:-} CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN} CLUSTER_NODES: ${CLUSTER_NODES:-http://node-a:9100,http://node-b:9100,http://node-c:9100} POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10 @@ -31,7 +38,7 @@ services: node-c: condition: service_healthy healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9000/ready"] + test: ["CMD", "wget", "-qO-", "--header", "X-Real-IP: 127.0.0.1", "http://127.0.0.1:9000/ready"] interval: 10s timeout: 4s retries: 3 diff --git a/compose.yaml b/compose.yaml index f3053d2..9c6004f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,8 +8,15 @@ services: S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env} S3_BUCKET: ${S3_BUCKET:-objects} S3_REGION: ${S3_REGION:-us-east-1} + S3_CREDENTIALS_FILE: ${S3_CREDENTIALS_FILE:-} MAX_OBJECT_BYTES: ${MAX_OBJECT_BYTES:-134217728} MAX_TOTAL_BYTES: ${MAX_TOTAL_BYTES:-2147483648} + PUBLIC_REQUESTS_PER_SECOND: ${PUBLIC_REQUESTS_PER_SECOND:-0} + PUBLIC_REQUEST_BURST: ${PUBLIC_REQUEST_BURST:-} + PUBLIC_BYTES_PER_SECOND: ${PUBLIC_BYTES_PER_SECOND:-0} + PUBLIC_BYTE_BURST: ${PUBLIC_BYTE_BURST:-} + PUBLIC_MAX_IN_FLIGHT_PER_IP: ${PUBLIC_MAX_IN_FLIGHT_PER_IP:-} + PUBLIC_TRUSTED_PROXY_IPS: ${PUBLIC_TRUSTED_PROXY_IPS:-} ports: - "127.0.0.1:${HOST_PORT:-9000}:9000" volumes: diff --git a/lib/LICENSE.hash4j b/lib/LICENSE.hash4j new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/lib/LICENSE.hash4j @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/hash4j-0.30.0.jar b/lib/hash4j-0.30.0.jar new file mode 100644 index 0000000..547d7fd Binary files /dev/null and b/lib/hash4j-0.30.0.jar differ diff --git a/scripts/objectstore b/scripts/objectstore index 3418433..111bb24 100644 --- a/scripts/objectstore +++ b/scripts/objectstore @@ -1,18 +1,18 @@ #!/bin/sh if [ "${1:-}" = "cluster-repair" ]; then shift - exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterRepair "$@" + exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterRepair "$@" fi if [ "${1:-}" = "cluster-gc" ]; then shift - exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterGc "$@" + exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterGc "$@" fi if [ "${1:-}" = "cluster-migrate" ]; then shift - exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterMigrate "$@" + exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterMigrate "$@" fi if [ "${1:-}" = "cluster-join" ]; then shift - exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterJoin "$@" + exec java -XX:MaxRAMPercentage=70 --add-modules java.net.http -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterJoin "$@" fi -exec java -XX:MaxRAMPercentage=70 -cp /app cloud.lunarsky.store.Cli "$@" +exec java -XX:MaxRAMPercentage=70 -cp /app:/app/hash4j.jar cloud.lunarsky.store.Cli "$@" diff --git a/scripts/test-cluster-http.py b/scripts/test-cluster-http.py index 2fb19b7..48d169c 100644 --- a/scripts/test-cluster-http.py +++ b/scripts/test-cluster-http.py @@ -4,11 +4,13 @@ import base64 import hashlib import hmac import http.client +import json import pathlib import re import sys import urllib.parse import zlib +import xml.etree.ElementTree as ET values = dict(line.strip().split("=", 1) for line in pathlib.Path(sys.argv[1]).read_text().splitlines() @@ -53,12 +55,42 @@ def request(method, path, body=b"", extra=None): connection.close() +def anonymous(method, path): + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + try: + connection.request(method, path) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + if len(sys.argv) > 2 and sys.argv[2] == "survivor": status, content, _ = request("GET", f"/{bucket}/cluster-test/survivor") assert status == 200 and content == b"acknowledged object survives node loss", (status, content) print("Cluster surviving-replica HTTP read passed") sys.exit(0) +if len(sys.argv) > 2 and sys.argv[2] == "acl": + path = f"/{bucket}/cluster-test/acl-multipart" + status, content, _ = request("POST", path + "?uploads", extra={"x-amz-acl": "public-read"}) + assert status == 200, (status, content) + upload_id = ET.fromstring(content).findtext("UploadId") + status, _, headers = request("PUT", path + f"?partNumber=1&uploadId={upload_id}", b"public part") + assert status == 200, status + completion = ("1" + + headers["etag"] + "").encode() + status, _, _ = request("POST", path + f"?uploadId={upload_id}", completion) + assert status == 200, status + status, content = anonymous("GET", path) + assert status == 200 and content == b"public part", (status, content) + status, _, _ = request("PUT", path, b"private replacement") + assert status == 200, status + status, _ = anonymous("GET", path) + assert status == 403, status + print("Cluster multipart ACL and replacement tests passed") + sys.exit(0) + if len(sys.argv) > 4 and sys.argv[2] == "status": key = urllib.parse.quote(sys.argv[3], safe="/") expected = int(sys.argv[4]) @@ -67,6 +99,35 @@ if len(sys.argv) > 4 and sys.argv[2] == "status": print(f"Cluster GET status passed: {status}") sys.exit(0) +if len(sys.argv) > 2 and sys.argv[2] == "version-survivor": + status, listing, _ = request("GET", "/version-bucket?versions") + assert status == 200, status + root = ET.fromstring(listing) + namespace = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"} + expected_etag = '"' + hashlib.md5(b"older cluster version").hexdigest() + '"' + versions = [version for version in root.findall("s3:Version", namespace) + if version.findtext("s3:ETag", namespaces=namespace) == expected_etag] + assert len(versions) == 1, listing + version_id = versions[0].findtext("s3:VersionId", namespaces=namespace) + status, content, _ = request("GET", "/version-bucket/note.txt?versionId=" + version_id) + assert status == 200 and content == b"older cluster version", (status, content) + multipart_versions = [version for version in root.findall("s3:Version", namespace) + if version.findtext("s3:Key", namespaces=namespace) == "multipart.txt"] + assert len(multipart_versions) == 1, listing + multipart_id = multipart_versions[0].findtext("s3:VersionId", namespaces=namespace) + status, content, _ = request("GET", "/version-bucket/multipart.txt?versionId=" + multipart_id) + assert status == 200 and content == b"retained multipart version", (status, content) + print("Cluster historical version survived repair and cleanup") + sys.exit(0) + + +status, content, headers = request("GET", "/_objectstore/capabilities") +capabilities = json.loads(content) +assert status == 200 and capabilities["schemaVersion"] == 1, (status, content) +assert capabilities["storageMode"] == "cluster", capabilities +assert "ListParts" in capabilities["operations"], capabilities +assert capabilities["limits"]["maxObjectBytes"] > 0, capabilities +assert headers.get_content_type() == "application/json", headers key = f"/{bucket}/cluster-test/http.txt" body = b"HTTP gateway integration test" @@ -85,6 +146,22 @@ assert status == 204, status status, _, _ = request("GET", key) assert status == 404, status +public_key = f"/{bucket}/cluster-test/public.txt" +status, _, _ = request("PUT", public_key, b"public object", {"x-amz-acl": "public-read"}) +assert status == 200, status +status, content = anonymous("GET", public_key) +assert status == 200 and content == b"public object", (status, content) +status, acl, _ = request("GET", public_key + "?acl") +assert status == 200 and b"AllUsers" in acl, (status, acl) +status, _, _ = request("PUT", public_key, b"private replacement") +assert status == 200, status +status, _ = anonymous("GET", public_key) +assert status == 403, status +status, _, _ = request("PUT", public_key + "?acl", extra={"x-amz-acl": "public-read"}) +assert status == 200, status +status, content = anonymous("GET", public_key) +assert status == 200 and content == b"private replacement", (status, content) + copy_source = f"/{bucket}/cluster-test/copy-source.txt" copy_target = f"/{bucket}/cluster-test/copied.txt" body = b"cluster copy and checksum test" @@ -95,6 +172,9 @@ status, _, headers = request("PUT", copy_source, body, "x-amz-checksum-crc32": crc32, "x-amz-sdk-checksum-algorithm": "CRC32"}) assert status == 200 and headers["x-amz-checksum-crc32"] == crc32, status +status, _, headers = request("HEAD", copy_source, + extra={"x-amz-checksum-mode": "ENABLED"}) +assert status == 200 and headers["x-amz-checksum-crc32"] == crc32, status status, content, _ = request("PUT", copy_source, body, {"content-md5": base64.b64encode(bytes(16)).decode()}) assert status == 400 and b"BadDigest" in content, (status, content) @@ -104,13 +184,99 @@ status, content, _ = request("PUT", copy_target, extra={"x-amz-copy-source": cop assert status == 200 and b"" in content, (status, content) status, content, headers = request("GET", copy_target) assert status == 200 and content == body and headers["content-type"] == "text/plain", (status, content) +status, _, headers = request("HEAD", copy_target, extra={"x-amz-checksum-mode": "ENABLED"}) +assert status == 200 and headers["x-amz-checksum-crc32"] == crc32, status status, _, _ = request("DELETE", copy_source) assert status == 204, status status, _, _ = request("DELETE", copy_target) assert status == 204, status +attribute_key = f"/{bucket}/cluster-test/attributes.txt" +status, _, _ = request("PUT", attribute_key, b"cluster attributes", + {"x-amz-meta-project": "LunarSky", "x-amz-tagging": "stage=one"}) +assert status == 200, status +status, content, headers = request("GET", attribute_key) +assert status == 200 and content == b"cluster attributes", (status, content) +assert headers["x-amz-meta-project"] == "LunarSky" and headers["x-amz-tagging-count"] == "1" +status, content, _ = request("GET", attribute_key + "?tagging") +assert status == 200 and b"stageone" in content, (status, content) +replacement = b"stagetwo" +status, _, _ = request("PUT", attribute_key + "?tagging", replacement) +assert status == 200, status +status, content, _ = request("GET", attribute_key + "?tagging") +assert status == 200 and b"two" in content, (status, content) +status, _, _ = request("DELETE", attribute_key + "?tagging") +assert status == 204, status +status, content, _ = request("GET", attribute_key + "?tagging") +assert status == 200 and b"" in content, (status, content) +status, content, _ = request("GET", "/") +assert status == 200 and f"{bucket}".encode() in content, (status, content) +status, _, _ = request("PUT", "/second-bucket") +assert status == 200, status +status, _, _ = request("PUT", "/second-bucket/one.txt", b"second bucket") +assert status == 200, status +status, content, _ = request("PUT", "/second-bucket/copied.txt", extra={"x-amz-copy-source": attribute_key}) +assert status == 200 and b"" in content, (status, content) +status, content, headers = request("GET", "/second-bucket/copied.txt") +assert status == 200 and content == b"cluster attributes", (status, content) +assert headers["x-amz-meta-project"] == "LunarSky" +status, _, _ = request("DELETE", attribute_key) +assert status == 204, status +status, content, _ = request("GET", "/second-bucket/one.txt") +assert status == 200 and content == b"second bucket", (status, content) +status, _, _ = request("DELETE", "/second-bucket") +assert status == 409, status +status, _, _ = request("DELETE", "/second-bucket/one.txt") +assert status == 204, status +status, _, _ = request("DELETE", "/second-bucket/copied.txt") +assert status == 204, status +status, _, _ = request("DELETE", "/second-bucket") +assert status == 204, status + +status, _, _ = request("PUT", "/version-bucket") +assert status == 200, status +version_key = "/version-bucket/note.txt" +status, _, _ = request("PUT", version_key, b"pre-versioning") +assert status == 200, status +versioning = b"Enabled" +status, _, _ = request("PUT", "/version-bucket?versioning", versioning) +assert status == 200, status +status, _, headers = request("PUT", version_key, b"older cluster version") +assert status == 200, status +old_version = headers["x-amz-version-id"] +status, _, headers = request("PUT", version_key, b"newer cluster version") +assert status == 200 and headers["x-amz-version-id"] != old_version, status +status, content, _ = request("GET", version_key + "?versionId=" + old_version) +assert status == 200 and content == b"older cluster version", (status, content) +status, _, headers = request("DELETE", version_key) +assert status == 204 and headers["x-amz-delete-marker"] == "true", status +status, content, _ = request("GET", version_key) +assert status == 404, (status, content) +status, content, _ = request("GET", "/version-bucket?versions") +assert status == 200 and b"" in content and old_version.encode() in content, (status, content) +versioned_multipart = "/version-bucket/multipart.txt" +status, content, _ = request("POST", versioned_multipart + "?uploads") +assert status == 200, (status, content) +versioned_upload = ET.fromstring(content).findtext("UploadId") +assert versioned_upload, content +versioned_part = b"retained multipart version" +status, _, headers = request("PUT", versioned_multipart + + f"?partNumber=1&uploadId={versioned_upload}", versioned_part) +assert status == 200, status +completion = ("1" + + headers["etag"] + "").encode() +status, _, headers = request("POST", versioned_multipart + f"?uploadId={versioned_upload}", completion) +assert status == 200 and headers.get("x-amz-version-id"), status +versioned_part_id = headers["x-amz-version-id"] +status, _, _ = request("DELETE", versioned_multipart) +assert status == 204, status +status, content, _ = request("GET", versioned_multipart + "?versionId=" + versioned_part_id) +assert status == 200 and content == versioned_part, (status, content) + multipart_key = f"/{bucket}/cluster-test/http-multipart.txt" -status, content, _ = request("POST", multipart_key + "?uploads") +status, content, _ = request("POST", multipart_key + "?uploads", + extra={"x-amz-meta-project": "multipart", "x-amz-tagging": "stage=upload", + "x-amz-acl": "public-read"}) assert status == 200, (status, content) match = re.search(rb"([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})", content[:8192]) @@ -134,8 +300,13 @@ completion = "" + "".join( status, content, _ = request("POST", multipart_key + f"?uploadId={upload_id}", completion.encode(), {"content-type": "application/xml"}) assert status == 200 and b"" in content, (status, content) -status, content, _ = request("GET", multipart_key) +status, content, headers = request("GET", multipart_key) assert status == 200 and content == b"hello world", (status, content) +status, public_content = anonymous("GET", multipart_key) +assert status == 200 and public_content == b"hello world", (status, public_content) +assert headers["x-amz-meta-project"] == "multipart" and headers["x-amz-tagging-count"] == "1" +status, _, headers = request("HEAD", multipart_key, extra={"x-amz-checksum-mode": "ENABLED"}) +assert status == 200 and len(base64.b64decode(headers["x-amz-checksum-crc64nvme"])) == 8, status status, content, _ = request("GET", f"/{bucket}?uploads&prefix=cluster-test%2Fhttp-multipart") assert status == 200 and upload_id.encode() not in content, (status, content) -print("Cluster HTTP tests passed: signed objects, copies, checksums, and multipart operations") +print("Cluster HTTP tests passed: signed objects, copies, checksums, multipart, and versioning") diff --git a/scripts/test-cluster.sh b/scripts/test-cluster.sh index a6570f9..61d0016 100644 --- a/scripts/test-cluster.sh +++ b/scripts/test-cluster.sh @@ -14,7 +14,7 @@ trap restore EXIT compose up -d --build run_phase() { compose exec -T gateway java --add-modules jdk.httpserver,java.net.http \ - -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterIntegrationTest "$1" + -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterIntegrationTest "$1" } wait_ready() { attempt=0 @@ -115,6 +115,7 @@ printf '%s\n' "$first_gc" | grep -q '^segments_deleted=0$' second_gc=$(compose run --rm -T gc --apply) printf '%s\n' "$second_gc" | grep -q '^segments_deleted=[1-9]' run_phase recovered +python3 scripts/test-cluster-http.py "$env_file" version-survivor run_phase verify-expanded backup_dir=$(mktemp -d) sh scripts/backup-cluster-metadata.sh "$env_file" "$backup_dir/metadata.dump" @@ -125,7 +126,7 @@ compose stop metadata compose run --rm -T --no-deps \ -e 'POSTGRES_JDBC_URL=jdbc:postgresql://metadata-recovery:5432/objectstore?connectTimeout=3&socketTimeout=10' \ --entrypoint java gateway --add-modules jdk.httpserver,java.net.http \ - -cp /app:/app/postgresql.jar cloud.lunarsky.store.ClusterIntegrationTest recovered + -cp /app:/app/postgresql.jar:/app/hash4j.jar cloud.lunarsky.store.ClusterIntegrationTest recovered compose start metadata wait_ready echo 'Cluster failure tests passed' diff --git a/scripts/test.sh b/scripts/test.sh index adb41e3..2debe09 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -2,10 +2,12 @@ set -eu cd "$(dirname "$0")/.." mkdir -p out/classes -javac --release 21 --add-modules jdk.httpserver,java.net.http -d out/classes \ +javac --release 21 --add-modules jdk.httpserver,java.net.http -cp lib/hash4j-0.30.0.jar -d out/classes \ src/cloud/lunarsky/store/*.java test/cloud/lunarsky/store/*.java -java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.StoreTest -java --add-modules jdk.httpserver -cp out/classes cloud.lunarsky.store.ConcurrencyTest -java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.HttpTest -java --add-modules jdk.httpserver,java.net.http -cp out/classes cloud.lunarsky.store.ClusterNodeTest -java -cp out/classes cloud.lunarsky.store.CliTest +java --add-modules jdk.httpserver -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.StoreTest +java --add-modules jdk.httpserver -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ConcurrencyTest +java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.HttpTest +java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClientLimitsTest +java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClusterNodeTest +java -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.CliTest +bash client/scripts/test.sh diff --git a/src/cloud/lunarsky/store/Acl.java b/src/cloud/lunarsky/store/Acl.java new file mode 100644 index 0000000..bd503db --- /dev/null +++ b/src/cloud/lunarsky/store/Acl.java @@ -0,0 +1,179 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.Headers; +import java.io.ByteArrayInputStream; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +final class Acl { + static final String ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers"; + static final String AUTHENTICATED_USERS = + "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"; + static final int READ = 1; + static final int WRITE = 2; + static final int READ_ACP = 4; + static final int WRITE_ACP = 8; + static final int FULL_CONTROL = READ | WRITE | READ_ACP | WRITE_ACP; + private static final Map PERMISSIONS = Map.of( + "READ", READ, "WRITE", WRITE, "READ_ACP", READ_ACP, + "WRITE_ACP", WRITE_ACP, "FULL_CONTROL", FULL_CONTROL); + + private Acl() {} + + static boolean allows(Map grants, String principal, String owner, int permission) { + if (owner.equals(principal)) return true; + if (principal != null && (bits(grants.get(principal)) & permission) == permission) return true; + if (principal != null && (bits(grants.get(AUTHENTICATED_USERS)) & permission) == permission) + return true; + return (bits(grants.get(ALL_USERS)) & permission) == permission; + } + + static void require(Map grants, String principal, String owner, int permission) { + if (!allows(grants, principal, owner, permission)) + throw new StoreException(403, "AccessDenied", "Access denied"); + } + + static Map fromHeaders(Headers headers, Set identities) { + String canned = SigV4.single(headers, "x-amz-acl"); + Set grantNames = Set.of("x-amz-grant-read", "x-amz-grant-write", + "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-grant-full-control"); + for (String name : headers.keySet()) { + String lower = name.toLowerCase(java.util.Locale.ROOT); + if (lower.startsWith("x-amz-grant-") && !grantNames.contains(lower)) + throw new StoreException(501, "NotImplemented", "Unsupported ACL grant header"); + if (canned != null && grantNames.contains(lower)) + throw new StoreException(400, "InvalidRequest", "Use either a canned ACL or explicit grants"); + } + TreeMap grants = new TreeMap<>(); + if (canned != null) { + if (!canned.equals("private") && !canned.equals("public-read") && + !canned.equals("authenticated-read") && + !canned.equals("bucket-owner-full-control")) + throw new StoreException(400, "InvalidArgument", "Unsupported canned ACL"); + if (canned.equals("public-read")) grants.put(ALL_USERS, READ); + if (canned.equals("authenticated-read")) grants.put(AUTHENTICATED_USERS, READ); + } + for (String name : new String[]{"read", "write", "read-acp", "write-acp", "full-control"}) { + String value = SigV4.single(headers, "x-amz-grant-" + name); + if (value == null) continue; + int permission = PERMISSIONS.get(name.replace('-', '_').toUpperCase(java.util.Locale.ROOT)); + for (String grant : value.split(",")) { + String entry = grant.trim(); + String principal; + if (entry.startsWith("id=\"") && entry.endsWith("\"")) + principal = entry.substring(4, entry.length() - 1); + else if (entry.startsWith("uri=\"") && entry.endsWith("\"")) + principal = entry.substring(5, entry.length() - 1); + else throw new StoreException(400, "InvalidArgument", "Invalid ACL grant"); + if (!identities.contains(principal) && !principal.equals(ALL_USERS) && + !principal.equals(AUTHENTICATED_USERS)) + throw new StoreException(400, "InvalidArgument", "Unknown ACL grantee"); + if (principal.equals(ALL_USERS) && permission != READ) + throw new StoreException(400, "InvalidArgument", "Only public read is supported"); + grants.merge(principal, permission, (left, right) -> left | right); + } + } + return encoded(grants); + } + + static Map validate(Map values, Set identities) { + if (values.size() > 64) throw new StoreException(400, "InvalidArgument", "Too many ACL grantees"); + TreeMap valid = new TreeMap<>(); + for (var entry : values.entrySet()) { + String principal = entry.getKey(); + int bits = bits(entry.getValue()); + if ((!identities.contains(principal) && !principal.equals(ALL_USERS) && + !principal.equals(AUTHENTICATED_USERS)) || + bits == 0 || (bits & ~FULL_CONTROL) != 0 || + principal.equals(ALL_USERS) && bits != READ) + throw new StoreException(400, "InvalidArgument", "Invalid ACL grantee or permission"); + valid.put(principal, Integer.toString(bits)); + } + return Map.copyOf(valid); + } + + static Map fromXml(byte[] body, String owner, Set identities) { + try { + var factory = javax.xml.parsers.DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setExpandEntityReferences(false); + var document = factory.newDocumentBuilder().parse(new ByteArrayInputStream(body)); + var root = document.getDocumentElement(); + if (!root.getLocalName().equals("AccessControlPolicy")) throw new IllegalArgumentException(); + var ownerNodes = root.getElementsByTagNameNS("*", "Owner"); + var lists = root.getElementsByTagNameNS("*", "AccessControlList"); + if (ownerNodes.getLength() != 1 || lists.getLength() != 1 || + !owner.equals(text((org.w3c.dom.Element) ownerNodes.item(0), "ID"))) + throw new IllegalArgumentException(); + TreeMap grants = new TreeMap<>(); + var nodes = ((org.w3c.dom.Element) lists.item(0)).getElementsByTagNameNS("*", "Grant"); + if (nodes.getLength() > 64) throw new IllegalArgumentException(); + for (int i = 0; i < nodes.getLength(); i++) { + var grant = (org.w3c.dom.Element) nodes.item(i); + var grantees = grant.getElementsByTagNameNS("*", "Grantee"); + if (grantees.getLength() != 1) throw new IllegalArgumentException(); + var grantee = (org.w3c.dom.Element) grantees.item(0); + String type = grantee.getAttributeNS("http://www.w3.org/2001/XMLSchema-instance", "type"); + String principal = switch (type) { + case "CanonicalUser" -> text(grantee, "ID"); + case "Group" -> text(grantee, "URI"); + default -> throw new IllegalArgumentException(); + }; + Integer permission = PERMISSIONS.get(text(grant, "Permission")); + if (permission == null) throw new IllegalArgumentException(); + grants.merge(principal, permission, (left, right) -> left | right); + } + grants.remove(owner); + return validate(encoded(grants), identities); + } catch (Exception error) { + throw new StoreException(400, "MalformedACLError", "Invalid access control policy"); + } + } + + private static String text(org.w3c.dom.Element element, String name) { + var nodes = element.getElementsByTagNameNS("*", name); + if (nodes.getLength() != 1) throw new IllegalArgumentException(); + return nodes.item(0).getTextContent().trim(); + } + + static String xml(Map grants, String owner) { + StringBuilder xml = new StringBuilder("") + .append(owner).append(""); + grant(xml, owner, "FULL_CONTROL", false); + for (var entry : new TreeMap<>(grants).entrySet()) { + if (entry.getKey().equals(owner)) continue; + int bits = bits(entry.getValue()); + if (bits == FULL_CONTROL) grant(xml, entry.getKey(), "FULL_CONTROL", + entry.getKey().equals(ALL_USERS) || entry.getKey().equals(AUTHENTICATED_USERS)); + else for (var permission : PERMISSIONS.entrySet()) + if (!permission.getKey().equals("FULL_CONTROL") && (bits & permission.getValue()) != 0) + grant(xml, entry.getKey(), permission.getKey(), + entry.getKey().equals(ALL_USERS) || entry.getKey().equals(AUTHENTICATED_USERS)); + } + return xml.append("").toString(); + } + + private static void grant(StringBuilder xml, String principal, String permission, boolean group) { + xml.append("" : "CanonicalUser\">") + .append(principal).append(group ? "" : "") + .append("").append(permission).append(""); + } + + private static Map encoded(Map grants) { + TreeMap values = new TreeMap<>(); + grants.forEach((key, value) -> values.put(key, Integer.toString(value))); + return Map.copyOf(values); + } + + private static int bits(String value) { + if (value == null) return 0; + try { return Integer.parseInt(value); } + catch (NumberFormatException error) { return 0; } + } +} diff --git a/src/cloud/lunarsky/store/AwsChunkedInputStream.java b/src/cloud/lunarsky/store/AwsChunkedInputStream.java new file mode 100644 index 0000000..9b4921f --- /dev/null +++ b/src/cloud/lunarsky/store/AwsChunkedInputStream.java @@ -0,0 +1,175 @@ +package cloud.lunarsky.store; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.HexFormat; +import java.util.zip.CRC32; +import java.util.zip.CRC32C; +import java.util.zip.Checksum; + +final class AwsChunkedInputStream extends FilterInputStream { + private static final String EMPTY_HASH = SigV4.hex(SigV4.hash(new byte[0])); + private final SigV4.Verified authorization; + private final long decodedLength; + private final String trailerName; + private final MessageDigest chunkHash; + private final MessageDigest trailerHash; + private final Checksum trailerCrc; + private final XxHashes trailerXxhash; + private long decoded; + private long chunkLeft; + private String suppliedSignature; + private String previousSignature; + private boolean finished; + private String trailerValue; + + AwsChunkedInputStream(InputStream input, SigV4.Verified authorization, + long decodedLength, String trailerName) { + super(input); + this.authorization = authorization; + this.decodedLength = decodedLength; + this.trailerName = trailerName; + this.previousSignature = authorization.signature(); + try { this.chunkHash = MessageDigest.getInstance("SHA-256"); } + catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); } + this.trailerCrc = trailerName == null ? null : switch (trailerName) { + case "x-amz-checksum-crc32" -> new CRC32(); + case "x-amz-checksum-crc32c" -> new CRC32C(); + case "x-amz-checksum-crc64nvme" -> new Crc64Nvme(); + default -> null; + }; + this.trailerXxhash = trailerName != null && java.util.Set.of("x-amz-checksum-xxhash64", + "x-amz-checksum-xxhash3", "x-amz-checksum-xxhash128").contains(trailerName) + ? new XxHashes(trailerName.substring("x-amz-checksum-".length()).toUpperCase(java.util.Locale.ROOT)) + : null; + String algorithm = trailerName == null ? null : switch (trailerName) { + case "x-amz-checksum-sha1" -> "SHA-1"; + case "x-amz-checksum-sha256" -> "SHA-256"; + case "x-amz-checksum-sha512" -> "SHA-512"; + case "x-amz-checksum-md5" -> "MD5"; + default -> null; + }; + if (trailerName != null && trailerCrc == null && trailerXxhash == null && algorithm == null) + throw new StoreException(501, "NotImplemented", "Checksum trailer is unsupported"); + try { this.trailerHash = algorithm == null ? null : MessageDigest.getInstance(algorithm); } + catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); } + } + + String trailerValue() { return trailerValue; } + + @Override public int read() throws IOException { + byte[] one = new byte[1]; + int count; + do { + count = read(one, 0, 1); + } while (count == 0); + return count < 0 ? -1 : one[0] & 255; + } + + @Override public int read(byte[] bytes, int offset, int length) throws IOException { + java.util.Objects.checkFromIndexSize(offset, length, bytes.length); + if (length == 0) return 0; + if (finished) return -1; + if (chunkLeft == 0) nextChunk(); + if (finished) return -1; + int count = in.read(bytes, offset, (int) Math.min(length, chunkLeft)); + if (count < 0) throw invalid("Incomplete signed chunk"); + if (count == 0) return 0; + chunkHash.update(bytes, offset, count); + if (trailerCrc != null) trailerCrc.update(bytes, offset, count); + if (trailerHash != null) trailerHash.update(bytes, offset, count); + if (trailerXxhash != null) trailerXxhash.update(bytes, offset, count); + chunkLeft -= count; + decoded += count; + if (chunkLeft == 0) { + if (!line().isEmpty()) throw invalid("Missing signed chunk separator"); + finishChunk(); + } + return count; + } + + private void nextChunk() throws IOException { + String line = line(); + int separator = line.indexOf(";chunk-signature="); + if (separator < 1 || separator != line.lastIndexOf(";chunk-signature=")) + throw invalid("Invalid signed chunk header"); + String size = line.substring(0, separator); + suppliedSignature = line.substring(separator + 17); + if (!size.matches("[0-9a-fA-F]{1,16}") || !suppliedSignature.matches("[0-9a-f]{64}")) + throw invalid("Invalid signed chunk header"); + try { chunkLeft = Long.parseUnsignedLong(size, 16); } + catch (NumberFormatException error) { throw invalid("Invalid signed chunk size"); } + if (chunkLeft > decodedLength - decoded) throw invalid("Signed chunks exceed decoded length"); + chunkHash.reset(); + if (chunkLeft == 0) { + finishChunk(); + if (decoded != decodedLength) throw invalid("Decoded length mismatch"); + if (trailerName == null) { + if (!line().isEmpty()) throw invalid("Invalid signed chunk ending"); + } else { + String trailer = line(); + if (!trailer.startsWith(trailerName + ":")) throw invalid("Missing signed checksum trailer"); + trailerValue = trailer.substring(trailerName.length() + 1); + byte[] actual; + if (trailerCrc != null) { + long value = trailerCrc.getValue(); + actual = new byte[trailerName.equals("x-amz-checksum-crc64nvme") ? 8 : 4]; + for (int i = actual.length - 1; i >= 0; i--) { + actual[i] = (byte) value; + value >>>= 8; + } + } else actual = trailerXxhash != null ? trailerXxhash.digest() : trailerHash.digest(); + if (!Base64.getEncoder().encodeToString(actual).equals(trailerValue)) + throw new StoreException(400, "BadDigest", "Checksum trailer mismatch"); + String signature = line(); + if (!signature.matches("x-amz-trailer-signature=[0-9a-f]{64}")) + throw invalid("Missing trailer signature"); + String toSign = "AWS4-HMAC-SHA256-TRAILER\n" + authorization.date() + "\n" + + authorization.scope() + "\n" + previousSignature + "\n" + + SigV4.hex(SigV4.hash((trailerName + ":" + trailerValue + "\n") + .getBytes(StandardCharsets.UTF_8))); + String expected = SigV4.hex(SigV4.hmac(authorization.signingKey(), toSign)); + if (!MessageDigest.isEqual(expected.getBytes(StandardCharsets.US_ASCII), + signature.substring(24).getBytes(StandardCharsets.US_ASCII))) + throw invalid("Trailer signature mismatch"); + if (!line().isEmpty()) throw invalid("Invalid trailer ending"); + } + if (in.read() != -1) throw invalid("Extra bytes after signed payload"); + finished = true; + } + } + + private void finishChunk() throws IOException { + String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + authorization.date() + "\n" + + authorization.scope() + "\n" + previousSignature + "\n" + EMPTY_HASH + "\n" + + SigV4.hex(chunkHash.digest()); + byte[] expected = SigV4.hmac(authorization.signingKey(), toSign); + if (!MessageDigest.isEqual(expected, HexFormat.of().parseHex(suppliedSignature))) + throw invalid("Signed chunk signature mismatch"); + previousSignature = suppliedSignature; + } + + private String line() throws IOException { + byte[] bytes = new byte[512]; + int count = 0; + while (count < bytes.length) { + int value = in.read(); + if (value < 0) throw invalid("Incomplete signed chunk framing"); + if (value == '\r') { + if (in.read() != '\n') throw invalid("Invalid signed chunk line ending"); + return new String(bytes, 0, count, StandardCharsets.US_ASCII); + } + if (value < 32 || value > 126) throw invalid("Invalid signed chunk line"); + bytes[count++] = (byte) value; + } + throw invalid("Signed chunk header is too long"); + } + + private static StoreException invalid(String message) { + return new StoreException(400, "InvalidRequest", message); + } +} diff --git a/src/cloud/lunarsky/store/Cli.java b/src/cloud/lunarsky/store/Cli.java index c021f58..e1fff3b 100644 --- a/src/cloud/lunarsky/store/Cli.java +++ b/src/cloud/lunarsky/store/Cli.java @@ -79,6 +79,15 @@ public final class Cli { try (var paths = Files.walk(objects)) { for (Path path : paths.filter(Files::isRegularFile).toList()) inspectObject(objects, path, verify, report); } + Path versions = root.resolve("versions"); + if (Files.isDirectory(versions)) { + try (var paths = Files.walk(versions)) { + for (Path path : paths.filter(Files::isRegularFile).toList()) { + if (!path.getFileName().toString().equals("manifest")) + inspectObject(versions, path, verify, report); + } + } + } Path multipart = root.resolve("multipart"); if (Files.isDirectory(multipart)) { try (var uploads = Files.list(multipart)) { @@ -100,7 +109,9 @@ public final class Cli { if (meta.key() == null) report.legacyObjects++; else { String id = SigV4.hex(SigV4.hash((meta.bucket() + "/" + meta.key()).getBytes(StandardCharsets.UTF_8))); - Path expected = objects.resolve(id.substring(0, 2)).resolve(id); + Path expected = objects.getFileName().toString().equals("versions") + ? objects.resolve(id.substring(0, 2)).resolve(id).resolve(path.getFileName()) + : objects.resolve(id.substring(0, 2)).resolve(id); if (!path.equals(expected)) report.problem("Mismatched object path: " + path); } if (size - record.headerLength() != meta.length()) { diff --git a/src/cloud/lunarsky/store/ClientLimits.java b/src/cloud/lunarsky/store/ClientLimits.java new file mode 100644 index 0000000..e0ecc72 --- /dev/null +++ b/src/cloud/lunarsky/store/ClientLimits.java @@ -0,0 +1,251 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.HttpExchange; +import java.io.FilterInputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +final class ClientLimits { + private static final int MAX_CLIENTS = 10_000; + private static final long IDLE_NANOS = 300_000_000_000L; + private final int requestsPerSecond; + private final int requestBurst; + private final long bytesPerSecond; + private final long byteBurst; + private final int maxInFlight; + private final Set trustedProxies; + private final Map clients = new HashMap<>(); + private long admissions; + + private ClientLimits(int requestsPerSecond, int requestBurst, long bytesPerSecond, + long byteBurst, int maxInFlight, Set trustedProxies) { + this.requestsPerSecond = requestsPerSecond; + this.requestBurst = requestBurst; + this.bytesPerSecond = bytesPerSecond; + this.byteBurst = byteBurst; + this.maxInFlight = maxInFlight; + this.trustedProxies = trustedProxies; + } + + static ClientLimits disabled() { + return new ClientLimits(0, 0, 0, 0, 0, Set.of()); + } + + static ClientLimits fromEnvironment(Map environment) { + int requests = number(environment, "PUBLIC_REQUESTS_PER_SECOND", 0); + int requestBurst = number(environment, "PUBLIC_REQUEST_BURST", requests); + long bytes = longNumber(environment, "PUBLIC_BYTES_PER_SECOND", 0); + long byteBurst = longNumber(environment, "PUBLIC_BYTE_BURST", bytes); + int inFlight = number(environment, "PUBLIC_MAX_IN_FLIGHT_PER_IP", + requests > 0 || bytes > 0 ? 8 : 0); + if (requests < 0 || requestBurst < 0 || bytes < 0 || byteBurst < 0 || inFlight < 0 || + requests > 0 && requestBurst < 1 || requests == 0 && requestBurst != 0 || + bytes > 0 && (byteBurst < 1 || byteBurst > 1_073_741_824L) || + bytes == 0 && byteBurst != 0 || + (requests > 0 || bytes > 0) && inFlight < 1) + throw new IllegalArgumentException("Invalid public client limits"); + Set proxies = new HashSet<>(); + String configured = environment.getOrDefault("PUBLIC_TRUSTED_PROXY_IPS", "").trim(); + if (!configured.isEmpty()) { + for (String item : configured.split(",", -1)) + proxies.add(numericAddress(item.trim()).getHostAddress()); + } + if (requests == 0 && bytes == 0 && inFlight == 0 && !proxies.isEmpty()) + throw new IllegalArgumentException("Trusted proxy IPs require public client limits"); + return new ClientLimits(requests, requestBurst, bytes, byteBurst, inFlight, Set.copyOf(proxies)); + } + + private static int number(Map environment, String name, int fallback) { + String value = environment.get(name); + if (value == null || value.isBlank()) return fallback; + try { return Integer.parseInt(value); } + catch (NumberFormatException error) { throw new IllegalArgumentException("Invalid " + name, error); } + } + + private static long longNumber(Map environment, String name, long fallback) { + String value = environment.get(name); + if (value == null || value.isBlank()) return fallback; + try { return Long.parseLong(value); } + catch (NumberFormatException error) { throw new IllegalArgumentException("Invalid " + name, error); } + } + + private static InetAddress numericAddress(String value) { + try { + if (value.matches("[0-9]{1,3}(\\.[0-9]{1,3}){3}")) { + String[] parts = value.split("\\."); + byte[] octets = new byte[4]; + for (int i = 0; i < 4; i++) { + int octet = Integer.parseInt(parts[i]); + if (octet > 255) throw new IllegalArgumentException("Invalid IP address"); + octets[i] = (byte) octet; + } + return InetAddress.getByAddress(octets); + } + if (value.contains(":") && value.matches("[0-9A-Fa-f:.]+")) + return InetAddress.getByName(value); + } catch (UnknownHostException error) { + throw new IllegalArgumentException("Invalid IP address", error); + } + throw new IllegalArgumentException("Expected numeric IP address"); + } + + private String address(HttpExchange exchange) { + InetAddress peer = exchange.getRemoteAddress().getAddress(); + String peerAddress = peer.getHostAddress(); + if (!trustedProxies.contains(peerAddress)) return peerAddress; + var values = exchange.getRequestHeaders().get("X-Real-IP"); + if (values == null || values.size() != 1) + throw new StoreException(400, "InvalidRequest", "Trusted proxy must supply one X-Real-IP address"); + try { return numericAddress(values.getFirst()).getHostAddress(); } + catch (IllegalArgumentException error) { + throw new StoreException(400, "InvalidRequest", "Trusted proxy supplied an invalid client address"); + } + } + + Client enter(HttpExchange exchange) { + if (requestsPerSecond == 0 && bytesPerSecond == 0 && maxInFlight == 0) return null; + String path = exchange.getRequestURI().getRawPath(); + if (exchange.getRemoteAddress().getAddress().isLoopbackAddress() && + exchange.getRequestHeaders().get("X-Real-IP") == null && + path.equals("/health")) return null; + String address = address(exchange); + Client client; + synchronized (this) { + long now = System.nanoTime(); + if (++admissions % 1024 == 0 || clients.size() >= MAX_CLIENTS) + clients.entrySet().removeIf(entry -> entry.getValue().inFlight == 0 && + now - entry.getValue().lastSeen > IDLE_NANOS); + client = clients.get(address); + if (client == null) { + if (clients.size() >= MAX_CLIENTS) + throw new StoreException(503, "SlowDown", "Client limit table is full"); + client = new Client(now, requestBurst, byteBurst); + clients.put(address, client); + } + refill(client, now); + client.lastSeen = now; + if (maxInFlight > 0 && client.inFlight >= maxInFlight) + throw new StoreException(503, "SlowDown", "Too many concurrent requests from this client"); + if (requestsPerSecond > 0 && client.requestTokens < 1) + throw new StoreException(503, "SlowDown", "Client request rate exceeded"); + if (requestsPerSecond > 0) client.requestTokens--; + client.inFlight++; + } + if (bytesPerSecond > 0) { + try { + exchange.setStreams(new LimitedInput(exchange.getRequestBody(), client), + new LimitedOutput(exchange.getResponseBody(), client)); + } catch (RuntimeException error) { + leave(client); + throw error; + } + } + return client; + } + + synchronized void leave(Client client) { + if (client != null) { + client.inFlight--; + client.lastSeen = System.nanoTime(); + } + } + + private void refill(Client client, long now) { + double seconds = Math.max(0, now - client.lastRefill) / 1_000_000_000.0; + if (requestsPerSecond > 0) + client.requestTokens = Math.min(requestBurst, client.requestTokens + seconds * requestsPerSecond); + if (bytesPerSecond > 0) + client.byteTokens = Math.min(byteBurst, client.byteTokens + seconds * bytesPerSecond); + client.lastRefill = now; + } + + private void pace(Client client, int count) throws IOException { + while (true) { + long wait; + synchronized (this) { + refill(client, System.nanoTime()); + if (client.byteTokens >= count) { + client.byteTokens -= count; + return; + } + wait = Math.max(1_000_000L, + (long) Math.ceil((count - client.byteTokens) * 1_000_000_000.0 / bytesPerSecond)); + } + try { Thread.sleep(Math.min(wait / 1_000_000L + 1, 1000)); } + catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IOException("Transfer interrupted while waiting for client bandwidth", error); + } + } + } + + private int chunk() { return (int) Math.min(16_384, byteBurst); } + + static final class Client { + private long lastSeen; + private long lastRefill; + private double requestTokens; + private double byteTokens; + private int inFlight; + + private Client(long now, int requestBurst, long byteBurst) { + lastSeen = now; + lastRefill = now; + requestTokens = requestBurst; + byteTokens = byteBurst; + } + } + + private final class LimitedInput extends FilterInputStream { + private final Client client; + + private LimitedInput(java.io.InputStream input, Client client) { + super(input); + this.client = client; + } + + @Override public int read() throws IOException { + int value = in.read(); + if (value >= 0) pace(client, 1); + return value; + } + + @Override public int read(byte[] bytes, int offset, int length) throws IOException { + int count = in.read(bytes, offset, Math.min(length, chunk())); + if (count > 0) pace(client, count); + return count; + } + } + + private final class LimitedOutput extends FilterOutputStream { + private final Client client; + + private LimitedOutput(java.io.OutputStream output, Client client) { + super(output); + this.client = client; + } + + @Override public void write(int value) throws IOException { + pace(client, 1); + out.write(value); + } + + @Override public void write(byte[] bytes, int offset, int length) throws IOException { + java.util.Objects.checkFromIndexSize(offset, length, bytes.length); + int left = length; + while (left > 0) { + int count = Math.min(left, chunk()); + pace(client, count); + out.write(bytes, offset, count); + offset += count; + left -= count; + } + } + } +} diff --git a/src/cloud/lunarsky/store/ClusterStore.java b/src/cloud/lunarsky/store/ClusterStore.java index bfdc04e..bff7e8b 100644 --- a/src/cloud/lunarsky/store/ClusterStore.java +++ b/src/cloud/lunarsky/store/ClusterStore.java @@ -23,13 +23,15 @@ import java.util.HexFormat; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; final class ClusterStore implements ObjectStorage, MultipartStorage { private record Segment(UUID id, int length, byte[] hash, List replicas) {} private record RepairTarget(UUID id, int part, int ordinal, long version, Segment segment) {} - private record Upload(String contentType) {} + private record Upload(String contentType, Map userMetadata, + Map tags, Map acl) {} private record StoredPart(long length, String etag, List segments) {} record RepairReport(int scanned, int restored, int rebalanced, int underReplicated, int unrecoverable) {} record GcReport(int scanned, int eligible, int deleted, int unavailableNodes) {} @@ -59,6 +61,168 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { private Connection connect() throws SQLException { return DriverManager.getConnection(jdbcUrl, user, password); } + @Override public Limits limits() { return new Limits(maxObject, maxTotal); } + + @Override public void ensureBucket(String bucket) throws IOException { + try { bucket(bucket); } + catch (StoreException error) { + if (error.status != 404) throw error; + try { createBucket(bucket); } + catch (StoreException created) { + if (created.status != 409) throw created; + } + } + } + + @Override public Bucket bucket(String name) throws IOException { + try (Connection connection = connect(); PreparedStatement query = connection.prepareStatement( + "SELECT created_at, versioning_state, acl FROM cluster_buckets WHERE name=?")) { + query.setString(1, name); + try (ResultSet result = query.executeQuery()) { + if (!result.next()) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + return new Bucket(name, result.getLong(1), VersioningState.valueOf(result.getString(2)), + ObjectAttributes.decode(result.getBytes(3))); + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public List buckets() throws IOException { + List result = new ArrayList<>(); + try (Connection connection = connect(); var query = connection.createStatement(); + ResultSet rows = query.executeQuery("SELECT name, created_at, versioning_state, acl FROM cluster_buckets ORDER BY name")) { + while (rows.next()) result.add(new Bucket(rows.getString(1), rows.getLong(2), + VersioningState.valueOf(rows.getString(3)), ObjectAttributes.decode(rows.getBytes(4)))); + return result; + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public void setVersioning(String bucket, VersioningState state) throws IOException { + if (state == VersioningState.NEVER) + throw new StoreException(400, "InvalidArgument", "Versioning cannot be disabled after it is enabled"); + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try { + lockUsage(connection, bucket); + VersioningState old = versioningState(connection, bucket); + if (old == VersioningState.NEVER && state == VersioningState.SUSPENDED) + throw new StoreException(400, "InvalidArgument", "Enable versioning before suspending it"); + try (PreparedStatement update = connection.prepareStatement( + "UPDATE cluster_buckets SET versioning_state=? WHERE name=?")) { + update.setString(1, state.name()); + update.setString(2, bucket); + update.executeUpdate(); + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public void setBucketAcl(String bucket, Map acl) throws IOException { + try (Connection connection = connect(); PreparedStatement update = connection.prepareStatement( + "UPDATE cluster_buckets SET acl=? WHERE name=?")) { + update.setBytes(1, ObjectAttributes.encode(acl, 2048)); + update.setString(2, bucket); + if (update.executeUpdate() == 0) + throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + } catch (SQLException error) { throw databaseError(error); } + } + + private static VersioningState versioningState(Connection connection, String bucket) throws SQLException { + try (PreparedStatement query = connection.prepareStatement( + "SELECT versioning_state FROM cluster_buckets WHERE name=? FOR UPDATE")) { + query.setString(1, bucket); + try (ResultSet result = query.executeQuery()) { + if (!result.next()) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + return VersioningState.valueOf(result.getString(1)); + } + } + } + + private static VersioningState readVersioningState(Connection connection, String bucket) throws SQLException { + try (PreparedStatement query = connection.prepareStatement( + "SELECT versioning_state FROM cluster_buckets WHERE name=?")) { + query.setString(1, bucket); + try (ResultSet result = query.executeQuery()) { + if (!result.next()) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + return VersioningState.valueOf(result.getString(1)); + } + } + } + + @Override public void createBucket(String name) throws IOException { + if (!name.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]")) + throw new StoreException(400, "InvalidBucketName", "Invalid bucket name"); + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try { + try (var statement = connection.createStatement()) { + statement.execute("SELECT pg_advisory_xact_lock(6834071092784)"); + } + try (var statement = connection.createStatement(); + ResultSet count = statement.executeQuery("SELECT count(*) FROM cluster_buckets")) { + count.next(); + if (count.getLong(1) >= 1000) + throw new StoreException(400, "TooManyBuckets", "Bucket limit reached"); + } + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_buckets (name, created_at) VALUES (?, ?) ON CONFLICT DO NOTHING")) { + insert.setString(1, name); + insert.setLong(2, Instant.now().toEpochMilli()); + if (insert.executeUpdate() == 0) + throw new StoreException(409, "BucketAlreadyOwnedByYou", "Bucket already exists"); + } + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_usage VALUES (?, 0)")) { + insert.setString(1, name); + insert.executeUpdate(); + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public void deleteBucket(String name) throws IOException { + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try { + lockUsage(connection, name); + try (PreparedStatement check = connection.prepareStatement( + "SELECT EXISTS (SELECT 1 FROM cluster_object_versions WHERE bucket=?) OR EXISTS " + + "(SELECT 1 FROM cluster_uploads WHERE bucket=?)")) { + check.setString(1, name); + check.setString(2, name); + try (ResultSet result = check.executeQuery()) { + result.next(); + if (result.getBoolean(1)) + throw new StoreException(409, "BucketNotEmpty", "Bucket contains objects or uploads"); + } + } + try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_buckets WHERE name=?")) { + delete.setString(1, name); + if (delete.executeUpdate() == 0) + throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + } + try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_usage WHERE bucket=?")) { + delete.setString(1, name); + delete.executeUpdate(); + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + private static void lockGc(Connection connection, boolean shared) throws SQLException { try (var statement = connection.createStatement()) { statement.execute("SELECT pg_advisory_lock" + (shared ? "_shared" : "") + "(6834071092783)"); @@ -66,26 +230,33 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } @Override public Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, - String checksum, boolean createOnly, String contentType) throws IOException { + String checksum, boolean createOnly, String contentType, + Map userMetadata, Map tags, + java.util.function.Supplier> checksums, + Map acl) throws IOException { validatePut(bucket, length, contentType); MessageDigest md5 = digest("MD5"); + Crc64Nvme crc64 = new Crc64Nvme(); Path staged = Files.createTempFile("objectstore-cluster-", ".pending"); try { - byte[] fullHash = stageInput(staged, input, length, expectedHash, checksum, md5); + byte[] fullHash = stageInput(staged, input, length, expectedHash, checksum, md5, crc64); + Map suppliedChecksums = checksums.get(); + Map storedChecksums = suppliedChecksums.isEmpty() ? + Map.of("x-amz-checksum-crc64nvme", crc64.encoded()) : Map.copyOf(suppliedChecksums); checkCapacity(bucket, key, length, createOnly); try (Connection connection = connect()) { lockGc(connection, true); List segments = uploadSegments(staged, length); Metadata metadata = new Metadata(length, Instant.now().toEpochMilli(), - HexFormat.of().formatHex(md5.digest()), fullHash, bucket, key, contentType); - persistObject(connection, metadata, segments, createOnly); - return metadata; + HexFormat.of().formatHex(md5.digest()), fullHash, bucket, key, contentType, + Map.copyOf(userMetadata), Map.copyOf(tags), null, storedChecksums, Map.copyOf(acl)); + return persistObject(connection, metadata, segments, createOnly); } catch (SQLException error) { throw databaseError(error); } } finally { Files.deleteIfExists(staged); } } - private void validatePut(String bucket, long length, String contentType) { - if (!configuredBucket.equals(bucket)) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + private void validatePut(String bucket, long length, String contentType) throws IOException { + bucket(bucket); if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required"); if (length > maxObject) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit"); if (!nodes.availableHostsAtLeast(2, testNodeDomains)) @@ -95,7 +266,7 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } private byte[] stageInput(Path staged, InputStream input, long length, String expectedHash, - String checksum, MessageDigest md5) throws IOException { + String checksum, MessageDigest md5, Crc64Nvme crc64) throws IOException { MessageDigest sha = digest("SHA-256"); try (OutputStream output = Files.newOutputStream(staged)) { byte[] buffer = new byte[65536]; @@ -106,13 +277,14 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { if (count == 0) continue; sha.update(buffer, 0, count); md5.update(buffer, 0, count); + crc64.update(buffer, 0, count); output.write(buffer, 0, count); remaining -= count; } } if (input.read() != -1) throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size"); byte[] fullHash = sha.digest(); - if (!HexFormat.of().formatHex(fullHash).equals(expectedHash)) + if (expectedHash != null && !HexFormat.of().formatHex(fullHash).equals(expectedHash)) throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch"); if (checksum != null && !Base64.getEncoder().encodeToString(fullHash).equals(checksum)) throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch"); @@ -121,18 +293,14 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { private void checkCapacity(String bucket, String key, long length, boolean createOnly) throws IOException { try (Connection connection = connect()) { + bucket(bucket); long previous = currentLength(connection, bucket, key); if (createOnly && previous >= 0) throw new StoreException(412, "PreconditionFailed", "Object already exists"); - try (PreparedStatement query = connection.prepareStatement("SELECT used_bytes FROM cluster_usage WHERE bucket=?")) { - query.setString(1, bucket); - try (ResultSet result = query.executeQuery()) { - if (!result.next()) throw new SQLException("Bucket quota row is missing"); - if (maxTotal - (result.getLong(1) - Math.max(0, previous)) - - stagedBytes(connection, bucket) < length) - throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); - } - } + long replaced = readVersioningState(connection, bucket) == VersioningState.ENABLED ? 0 : + nullVersionLength(connection, bucket, key); + if (maxTotal - (occupiedBytes(connection) - replaced) < length) + throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); } catch (SQLException error) { throw databaseError(error); } } @@ -170,8 +338,8 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { return segments; } - private void persistObject(Connection connection, Metadata metadata, List segments, - boolean createOnly) throws IOException { + private Metadata persistObject(Connection connection, Metadata metadata, List segments, + boolean createOnly) throws IOException { String bucket = metadata.bucket(), key = metadata.key(); long length = metadata.length(); UUID generation = UUID.randomUUID(); @@ -179,11 +347,12 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { connection.setAutoCommit(false); try { long used = lockUsage(connection, bucket); + VersioningState state = versioningState(connection, bucket); long previous = currentLength(connection, bucket, key); if (createOnly && previous >= 0) throw new StoreException(412, "PreconditionFailed", "Object already exists"); - if (maxTotal - (used - Math.max(0, previous)) - - stagedBytes(connection, bucket) < length) + long replaced = state == VersioningState.ENABLED ? 0 : nullVersionLength(connection, bucket, key); + if (maxTotal - (occupiedBytes(connection) - replaced) < length) throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); try (PreparedStatement insert = connection.prepareStatement( "INSERT INTO cluster_segments (generation, ordinal, segment_id, length, sha256, replicas, replica_ids) VALUES (?, ?, ?, ?, ?, 'v2', ?)")) { @@ -199,22 +368,9 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } insert.executeBatch(); } - try (PreparedStatement update = connection.prepareStatement( - "INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, length=EXCLUDED.length, modified=EXCLUDED.modified, etag=EXCLUDED.etag, sha256=EXCLUDED.sha256, content_type=EXCLUDED.content_type")) { - bindObject(update, metadata, generation); - update.executeUpdate(); - } - try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) { - update.setLong(1, used - Math.max(0, previous) + length); - update.setString(2, bucket); - update.executeUpdate(); - } - try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) { - delete.setString(1, bucket); - delete.setString(2, key); - delete.executeUpdate(); - } + Metadata stored = publishObject(connection, metadata, generation, used, replaced, state); connection.commit(); + return stored; } catch (SQLException | RuntimeException error) { connection.rollback(); if (error instanceof SQLException sql) throw databaseError(sql); @@ -223,8 +379,96 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } catch (SQLException error) { throw databaseError(error); } } - @Override public String create(String bucket, String key, String contentType) throws IOException { - if (!configuredBucket.equals(bucket)) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + private static long nullVersionLength(Connection connection, String bucket, String key) throws SQLException { + try (PreparedStatement query = connection.prepareStatement( + "SELECT length FROM cluster_object_versions WHERE bucket=? AND object_key=? AND version_id='null'")) { + query.setString(1, bucket); + query.setString(2, key); + try (ResultSet result = query.executeQuery()) { + return result.next() ? result.getLong(1) : 0; + } + } + } + + private Metadata publishObject(Connection connection, Metadata metadata, UUID generation, + long used, long replaced, VersioningState state) throws SQLException { + String bucket = metadata.bucket(), key = metadata.key(); + String id = state == VersioningState.ENABLED ? UUID.randomUUID().toString() : "null"; + if (id.equals("null")) { + try (PreparedStatement delete = connection.prepareStatement( + "DELETE FROM cluster_object_versions WHERE bucket=? AND object_key=? AND version_id='null'")) { + delete.setString(1, bucket); + delete.setString(2, key); + delete.executeUpdate(); + } + } + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_object_versions (bucket, object_key, version_id, delete_marker, generation, " + + "length, modified, etag, sha256, content_type, user_metadata, tags, checksum_metadata, acl) " + + "VALUES (?, ?, ?, false, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { + insert.setString(1, bucket); + insert.setString(2, key); + insert.setString(3, id); + insert.setObject(4, generation); + insert.setLong(5, metadata.length()); + insert.setLong(6, metadata.modified()); + insert.setString(7, metadata.etag()); + insert.setBytes(8, metadata.sha256()); + insert.setString(9, metadata.contentType()); + insert.setBytes(10, ObjectAttributes.encode(metadata.userMetadata(), 4096)); + insert.setBytes(11, ObjectAttributes.encode(metadata.tags(), 8192)); + insert.setBytes(12, ObjectAttributes.encode(metadata.checksums(), 512)); + insert.setBytes(13, ObjectAttributes.encode(metadata.acl(), 2048)); + insert.executeUpdate(); + } + setHead(connection, bucket, key, id); + writeCurrentObject(connection, metadata, generation); + try (PreparedStatement update = connection.prepareStatement( + "UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) { + update.setLong(1, used - replaced + metadata.length()); + update.setString(2, bucket); + update.executeUpdate(); + } + try (PreparedStatement delete = connection.prepareStatement( + "DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) { + delete.setString(1, bucket); + delete.setString(2, key); + delete.executeUpdate(); + } + return new Metadata(metadata.length(), metadata.modified(), metadata.etag(), metadata.sha256(), + bucket, key, metadata.contentType(), metadata.userMetadata(), metadata.tags(), + state == VersioningState.NEVER ? null : id, metadata.checksums(), metadata.acl()); + } + + private static void setHead(Connection connection, String bucket, String key, String id) throws SQLException { + try (PreparedStatement update = connection.prepareStatement( + "INSERT INTO cluster_object_heads VALUES (?, ?, ?) ON CONFLICT (bucket, object_key) " + + "DO UPDATE SET version_id=EXCLUDED.version_id")) { + update.setString(1, bucket); + update.setString(2, key); + update.setString(3, id); + update.executeUpdate(); + } + } + + private static void writeCurrentObject(Connection connection, Metadata metadata, + UUID generation) throws SQLException { + try (PreparedStatement update = connection.prepareStatement( + "INSERT INTO cluster_objects (bucket, object_key, generation, length, modified, etag, sha256, " + + "content_type, user_metadata, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + + "ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, " + + "length=EXCLUDED.length, modified=EXCLUDED.modified, etag=EXCLUDED.etag, " + + "sha256=EXCLUDED.sha256, content_type=EXCLUDED.content_type, " + + "user_metadata=EXCLUDED.user_metadata, tags=EXCLUDED.tags")) { + bindObject(update, metadata, generation); + update.executeUpdate(); + } + } + + @Override public String create(String bucket, String key, String contentType, + Map userMetadata, Map tags, + Map acl) throws IOException { + bucket(bucket); if (contentType == null || contentType.getBytes(java.nio.charset.StandardCharsets.UTF_8).length > 255) throw new StoreException(400, "InvalidArgument", "Invalid Content-Type"); UUID id = UUID.randomUUID(); @@ -241,12 +485,15 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } } try (PreparedStatement insert = connection.prepareStatement( - "INSERT INTO cluster_uploads VALUES (?, ?, ?, ?, ?)")) { + "INSERT INTO cluster_uploads (upload_id, bucket, object_key, content_type, created_at, user_metadata, tags, acl) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) { insert.setObject(1, id); insert.setString(2, bucket); insert.setString(3, key); insert.setString(4, contentType); insert.setLong(5, Instant.now().toEpochMilli()); + insert.setBytes(6, ObjectAttributes.encode(userMetadata, 4096)); + insert.setBytes(7, ObjectAttributes.encode(tags, 8192)); + insert.setBytes(8, ObjectAttributes.encode(acl, 2048)); insert.executeUpdate(); } connection.commit(); @@ -269,7 +516,7 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { Path staged = Files.createTempFile("objectstore-part-", ".pending"); MessageDigest md5 = digest("MD5"); try { - stageInput(staged, input, length, expectedHash, checksum, md5); + stageInput(staged, input, length, expectedHash, checksum, md5, new Crc64Nvme()); try (Connection connection = connect()) { lockGc(connection, true); List segments = uploadSegments(staged, length); @@ -279,7 +526,7 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { long used = lockUsage(connection, bucket); upload(connection, uploadId, bucket, key, true); long previous = partLength(connection, uploadId, number); - if (maxTotal - used - (stagedBytes(connection, bucket) - previous) < length) + if (maxTotal - (occupiedBytes(connection) - previous) < length) throw new StoreException(507, "InsufficientStorage", "Multipart staging limit reached"); try (PreparedStatement insert = connection.prepareStatement( "INSERT INTO cluster_upload_parts VALUES (?, ?, ?, ?, ?) ON CONFLICT (upload_id, part_number) DO UPDATE SET length=EXCLUDED.length, etag=EXCLUDED.etag, modified=EXCLUDED.modified")) { @@ -330,11 +577,12 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { connection.setAutoCommit(false); try { long used = lockUsage(connection, bucket); + VersioningState state = versioningState(connection, bucket); Upload upload = upload(connection, uploadId, bucket, key, true); - long staged = stagedBytes(connection, bucket); long uploadBytes = uploadLength(connection, uploadId); MessageDigest fullHash = digest("SHA-256"); MessageDigest etagHash = digest("MD5"); + Crc64Nvme crc64 = new Crc64Nvme(); List selected = new ArrayList<>(); long total = 0; int last = 0; @@ -353,6 +601,7 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { byte[] bytes = readableSegment(segment); if (bytes == null) throw new StoreException(503, "SlowDown", "A part has no verified replica"); fullHash.update(bytes); + crc64.update(bytes, 0, bytes.length); partHash.update(bytes); selected.add(segment); } @@ -361,12 +610,13 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { throw new StoreException(503, "SlowDown", "A part failed integrity verification"); etagHash.update(md5); } - long previous = currentLength(connection, bucket, key); - if (maxTotal - (used - Math.max(0, previous)) - (staged - uploadBytes) < total) + long replaced = state == VersioningState.ENABLED ? 0 : nullVersionLength(connection, bucket, key); + if (maxTotal - (occupiedBytes(connection) - replaced - uploadBytes) < total) throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); Metadata metadata = new Metadata(total, Instant.now().toEpochMilli(), HexFormat.of().formatHex(etagHash.digest()) + "-" + parts.size(), fullHash.digest(), - bucket, key, upload.contentType()); + bucket, key, upload.contentType(), upload.userMetadata(), upload.tags(), null, + Map.of("x-amz-checksum-crc64nvme", crc64.encoded()), upload.acl()); UUID generation = UUID.randomUUID(); try (PreparedStatement insert = connection.prepareStatement( "INSERT INTO cluster_segments (generation, ordinal, segment_id, length, sha256, replicas, replica_ids) VALUES (?, ?, ?, ?, ?, 'v2', ?)")) { @@ -382,27 +632,13 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } insert.executeBatch(); } - try (PreparedStatement update = connection.prepareStatement( - "INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, length=EXCLUDED.length, modified=EXCLUDED.modified, etag=EXCLUDED.etag, sha256=EXCLUDED.sha256, content_type=EXCLUDED.content_type")) { - bindObject(update, metadata, generation); - update.executeUpdate(); - } - try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) { - update.setLong(1, used - Math.max(0, previous) + total); - update.setString(2, bucket); - update.executeUpdate(); - } - try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) { - delete.setString(1, bucket); - delete.setString(2, key); - delete.executeUpdate(); - } + Metadata stored = publishObject(connection, metadata, generation, used, replaced, state); try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_uploads WHERE upload_id=?")) { delete.setObject(1, uploadId); delete.executeUpdate(); } connection.commit(); - return metadata; + return stored; } catch (SQLException | IOException | RuntimeException error) { connection.rollback(); if (error instanceof SQLException sql) throw databaseError(sql); @@ -460,7 +696,7 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } @Override public List listUploads(String bucket, String prefix) throws IOException { - if (!configuredBucket.equals(bucket)) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + bucket(bucket); List uploads = new ArrayList<>(); try (Connection connection = connect(); PreparedStatement query = connection.prepareStatement( "SELECT upload_id, object_key, created_at FROM cluster_uploads WHERE bucket=?")) { @@ -494,6 +730,10 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } @Override public OpenObject open(String bucket, String key) throws IOException { + return open(bucket, key, null); + } + + @Override public OpenObject open(String bucket, String key, String versionId) throws IOException { try (Connection connection = connect()) { connection.setAutoCommit(false); connection.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ); @@ -502,14 +742,32 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { Metadata metadata; UUID generation; try (PreparedStatement query = connection.prepareStatement( - "SELECT generation, length, modified, etag, sha256, content_type FROM cluster_objects WHERE bucket=? AND object_key=?")) { + versionId == null ? + "SELECT v.version_id, v.delete_marker, v.generation, v.length, v.modified, v.etag, " + + "v.sha256, v.content_type, v.user_metadata, v.tags, v.checksum_metadata, v.acl FROM cluster_object_heads h " + + "JOIN cluster_object_versions v ON v.bucket=h.bucket AND v.object_key=h.object_key " + + "AND v.version_id=h.version_id WHERE h.bucket=? AND h.object_key=?" : + "SELECT version_id, delete_marker, generation, length, modified, etag, sha256, " + + "content_type, user_metadata, tags, checksum_metadata, acl FROM cluster_object_versions WHERE bucket=? " + + "AND object_key=? AND version_id=?")) { query.setString(1, bucket); query.setString(2, key); + if (versionId != null) query.setString(3, versionId); try (ResultSet result = query.executeQuery()) { - if (!result.next()) throw new StoreException(404, "NoSuchKey", "Object not found"); - generation = (UUID) result.getObject(1); - metadata = new Metadata(result.getLong(2), result.getLong(3), result.getString(4), - result.getBytes(5), bucket, key, result.getString(6)); + if (!result.next()) throw new StoreException(404, + versionId == null ? "NoSuchKey" : "NoSuchVersion", "Object version not found"); + if (result.getBoolean(2)) + throw StoreException.deletedVersion(result.getString(1), result.getLong(5), + versionId != null); + generation = (UUID) result.getObject(3); + String storedId = result.getString(1); + boolean unversioned = versionId == null && storedId.equals("null") && + readVersioningState(connection, bucket) == VersioningState.NEVER; + metadata = new Metadata(result.getLong(4), result.getLong(5), result.getString(6), + result.getBytes(7), bucket, key, result.getString(8), + ObjectAttributes.decode(result.getBytes(9)), ObjectAttributes.decode(result.getBytes(10)), + unversioned ? null : storedId, ObjectAttributes.decode(result.getBytes(11)), + ObjectAttributes.decode(result.getBytes(12))); } } List parts = new ArrayList<>(); @@ -539,30 +797,253 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } @Override public void delete(String bucket, String key) throws IOException { + delete(bucket, key, null); + } + + @Override public DeleteResult delete(String bucket, String key, String versionId) throws IOException { try (Connection connection = connect()) { connection.setAutoCommit(false); try { long used = lockUsage(connection, bucket); - long previous = currentLength(connection, bucket, key); - try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_objects WHERE bucket=? AND object_key=?")) { - delete.setString(1, bucket); - delete.setString(2, key); - delete.executeUpdate(); + VersioningState state = versioningState(connection, bucket); + long removedLength = 0; + boolean removedMarker = false; + String resultId = null; + if (versionId != null) { + try (PreparedStatement query = connection.prepareStatement( + "SELECT delete_marker, length FROM cluster_object_versions WHERE bucket=? " + + "AND object_key=? AND version_id=?")) { + query.setString(1, bucket); + query.setString(2, key); + query.setString(3, versionId); + try (ResultSet result = query.executeQuery()) { + if (!result.next()) + throw new StoreException(404, "NoSuchVersion", "Object version not found"); + removedMarker = result.getBoolean(1); + removedLength = removedMarker ? 0 : result.getLong(2); + } + } + removeVersionRow(connection, bucket, key, versionId); + refreshCurrent(connection, bucket, key); + resultId = versionId; + } else if (state == VersioningState.NEVER) { + removedLength = nullVersionLength(connection, bucket, key); + removeVersionRow(connection, bucket, key, "null"); + refreshCurrent(connection, bucket, key); + } else { + resultId = state == VersioningState.ENABLED ? UUID.randomUUID().toString() : "null"; + if (state == VersioningState.SUSPENDED) { + removedLength = nullVersionLength(connection, bucket, key); + removeVersionRow(connection, bucket, key, "null"); + } + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_object_versions (bucket, object_key, version_id, " + + "delete_marker, modified) VALUES (?, ?, ?, true, ?)")) { + insert.setString(1, bucket); + insert.setString(2, key); + insert.setString(3, resultId); + insert.setLong(4, Instant.now().toEpochMilli()); + insert.executeUpdate(); + } + setHead(connection, bucket, key, resultId); + removeCurrentObject(connection, bucket, key); + recordTombstone(connection, bucket, key); } try (PreparedStatement update = connection.prepareStatement( - "INSERT INTO cluster_tombstones VALUES (?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, deleted_at=EXCLUDED.deleted_at")) { - update.setString(1, bucket); - update.setString(2, key); - update.setObject(3, UUID.randomUUID()); - update.setLong(4, Instant.now().toEpochMilli()); + "UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) { + update.setLong(1, used - removedLength); + update.setString(2, bucket); update.executeUpdate(); } - if (previous >= 0) { - try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) { - update.setLong(1, used - previous); - update.setString(2, bucket); - update.executeUpdate(); + connection.commit(); + return new DeleteResult(resultId, versionId == null && state != VersioningState.NEVER || removedMarker); + } catch (SQLException | IOException | RuntimeException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + if (error instanceof IOException io) throw io; + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + + private static void removeVersionRow(Connection connection, String bucket, String key, + String versionId) throws SQLException { + try (PreparedStatement delete = connection.prepareStatement( + "DELETE FROM cluster_object_versions WHERE bucket=? AND object_key=? AND version_id=?")) { + delete.setString(1, bucket); + delete.setString(2, key); + delete.setString(3, versionId); + delete.executeUpdate(); + } + } + + private static void removeCurrentObject(Connection connection, String bucket, String key) throws SQLException { + try (PreparedStatement delete = connection.prepareStatement( + "DELETE FROM cluster_objects WHERE bucket=? AND object_key=?")) { + delete.setString(1, bucket); + delete.setString(2, key); + delete.executeUpdate(); + } + } + + private static void recordTombstone(Connection connection, String bucket, String key) throws SQLException { + try (PreparedStatement update = connection.prepareStatement( + "INSERT INTO cluster_tombstones VALUES (?, ?, ?, ?) ON CONFLICT (bucket, object_key) " + + "DO UPDATE SET generation=EXCLUDED.generation, deleted_at=EXCLUDED.deleted_at")) { + update.setString(1, bucket); + update.setString(2, key); + update.setObject(3, UUID.randomUUID()); + update.setLong(4, Instant.now().toEpochMilli()); + update.executeUpdate(); + } + } + + private static void refreshCurrent(Connection connection, String bucket, String key) + throws SQLException, IOException { + try (PreparedStatement latest = connection.prepareStatement( + "SELECT version_id, delete_marker, generation, length, modified, etag, sha256, " + + "content_type, user_metadata, tags FROM cluster_object_versions WHERE bucket=? " + + "AND object_key=? ORDER BY sequence DESC LIMIT 1")) { + latest.setString(1, bucket); + latest.setString(2, key); + try (ResultSet result = latest.executeQuery()) { + if (!result.next()) { + try (PreparedStatement delete = connection.prepareStatement( + "DELETE FROM cluster_object_heads WHERE bucket=? AND object_key=?")) { + delete.setString(1, bucket); + delete.setString(2, key); + delete.executeUpdate(); } + removeCurrentObject(connection, bucket, key); + recordTombstone(connection, bucket, key); + return; + } + setHead(connection, bucket, key, result.getString(1)); + if (result.getBoolean(2)) { + removeCurrentObject(connection, bucket, key); + recordTombstone(connection, bucket, key); + } else { + Metadata metadata = new Metadata(result.getLong(4), result.getLong(5), + result.getString(6), result.getBytes(7), bucket, key, result.getString(8), + ObjectAttributes.decode(result.getBytes(9)), ObjectAttributes.decode(result.getBytes(10))); + writeCurrentObject(connection, metadata, (UUID) result.getObject(3)); + try (PreparedStatement delete = connection.prepareStatement( + "DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) { + delete.setString(1, bucket); + delete.setString(2, key); + delete.executeUpdate(); + } + } + } + } + } + + @Override public Map tags(String bucket, String key) throws IOException { + return tags(bucket, key, null); + } + + @Override public Map tags(String bucket, String key, String versionId) throws IOException { + try (Connection connection = connect(); PreparedStatement query = connection.prepareStatement( + versionId == null ? "SELECT v.tags, v.delete_marker FROM cluster_object_heads h " + + "JOIN cluster_object_versions v ON v.bucket=h.bucket AND v.object_key=h.object_key " + + "AND v.version_id=h.version_id WHERE h.bucket=? AND h.object_key=?" : + "SELECT tags, delete_marker FROM cluster_object_versions WHERE bucket=? AND object_key=? " + + "AND version_id=?")) { + query.setString(1, bucket); + query.setString(2, key); + if (versionId != null) query.setString(3, versionId); + try (ResultSet result = query.executeQuery()) { + if (!result.next() || result.getBoolean(2)) + throw new StoreException(404, versionId == null ? "NoSuchKey" : "NoSuchVersion", + "Object version not found"); + return ObjectAttributes.decode(result.getBytes(1)); + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public void setTags(String bucket, String key, Map tags) throws IOException { + setTags(bucket, key, null, tags); + } + + @Override public void setTags(String bucket, String key, String versionId, + Map tags) throws IOException { + byte[] encoded = ObjectAttributes.encode(tags, 8192); + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try { + lockUsage(connection, bucket); + String selected = versionId; + if (selected == null) { + try (PreparedStatement head = connection.prepareStatement( + "SELECT version_id FROM cluster_object_heads WHERE bucket=? AND object_key=?")) { + head.setString(1, bucket); + head.setString(2, key); + try (ResultSet result = head.executeQuery()) { + if (!result.next()) throw new StoreException(404, "NoSuchKey", "Object not found"); + selected = result.getString(1); + } + } + } + try (PreparedStatement update = connection.prepareStatement( + "UPDATE cluster_object_versions SET tags=? WHERE bucket=? AND object_key=? " + + "AND version_id=? AND NOT delete_marker")) { + update.setBytes(1, encoded); + update.setString(2, bucket); + update.setString(3, key); + update.setString(4, selected); + if (update.executeUpdate() == 0) + throw new StoreException(404, versionId == null ? "NoSuchKey" : "NoSuchVersion", + "Object version not found"); + } + try (PreparedStatement update = connection.prepareStatement( + "UPDATE cluster_objects SET tags=? WHERE bucket=? AND object_key=? AND EXISTS " + + "(SELECT 1 FROM cluster_object_heads WHERE bucket=? AND object_key=? AND version_id=?)")) { + update.setBytes(1, encoded); + update.setString(2, bucket); + update.setString(3, key); + update.setString(4, bucket); + update.setString(5, key); + update.setString(6, selected); + update.executeUpdate(); + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + if (error instanceof SQLException sql) throw databaseError(sql); + throw error; + } + } catch (SQLException error) { throw databaseError(error); } + } + + @Override public void setObjectAcl(String bucket, String key, String versionId, + Map acl) throws IOException { + byte[] encoded = ObjectAttributes.encode(acl, 2048); + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try { + lockUsage(connection, bucket); + String selected = versionId; + if (selected == null) { + try (PreparedStatement head = connection.prepareStatement( + "SELECT version_id FROM cluster_object_heads WHERE bucket=? AND object_key=?")) { + head.setString(1, bucket); + head.setString(2, key); + try (ResultSet result = head.executeQuery()) { + if (!result.next()) throw new StoreException(404, "NoSuchKey", "Object not found"); + selected = result.getString(1); + } + } + } + try (PreparedStatement update = connection.prepareStatement( + "UPDATE cluster_object_versions SET acl=? WHERE bucket=? AND object_key=? " + + "AND version_id=? AND NOT delete_marker")) { + update.setBytes(1, encoded); + update.setString(2, bucket); + update.setString(3, key); + update.setString(4, selected); + if (update.executeUpdate() == 0) + throw new StoreException(404, versionId == null ? "NoSuchKey" : "NoSuchVersion", + "Object version not found"); } connection.commit(); } catch (SQLException | RuntimeException error) { @@ -592,6 +1073,61 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } catch (SQLException error) { throw databaseError(error); } } + @Override public VersionPage listVersions(String bucket, String prefix, String keyMarker, + String versionMarker, int maxKeys) throws IOException { + if (versionMarker != null && keyMarker == null) + throw new StoreException(400, "InvalidArgument", "Version marker requires a key marker"); + bucket(bucket); + if (maxKeys == 0) return new VersionPage(List.of(), null, null, false); + List page = new ArrayList<>(); + String nextKey = null, nextVersion = null; + boolean truncated = false; + try (Connection connection = connect()) { + connection.setAutoCommit(false); + try (PreparedStatement query = connection.prepareStatement( + "SELECT v.object_key, v.version_id, v.delete_marker, v.length, v.modified, v.etag, " + + "v.sha256, v.content_type, v.user_metadata, v.tags, h.version_id=v.version_id, " + + "v.checksum_metadata " + + "FROM cluster_object_versions v LEFT JOIN cluster_object_heads h ON " + + "h.bucket=v.bucket AND h.object_key=v.object_key WHERE v.bucket=? AND v.object_key>=? " + + "ORDER BY v.object_key, v.sequence DESC")) { + query.setString(1, bucket); + query.setString(2, keyMarker != null && keyMarker.compareTo(prefix) > 0 ? keyMarker : prefix); + query.setFetchSize(128); + try (ResultSet rows = query.executeQuery()) { + boolean pastMarker = versionMarker == null; + while (rows.next()) { + String key = rows.getString(1), id = rows.getString(2); + if (!key.startsWith(prefix)) break; + if (keyMarker != null && key.compareTo(keyMarker) < 0) continue; + if (keyMarker != null && key.compareTo(keyMarker) > 0) pastMarker = true; + if (keyMarker != null && key.equals(keyMarker)) { + if (versionMarker == null) continue; + if (!pastMarker) { + if (id.equals(versionMarker)) pastMarker = true; + continue; + } + } + if (page.size() == maxKeys) { + truncated = true; + break; + } + boolean marker = rows.getBoolean(3); + Metadata metadata = marker ? null : new Metadata(rows.getLong(4), rows.getLong(5), + rows.getString(6), rows.getBytes(7), bucket, key, rows.getString(8), + ObjectAttributes.decode(rows.getBytes(9)), ObjectAttributes.decode(rows.getBytes(10)), + id, ObjectAttributes.decode(rows.getBytes(12))); + page.add(new VersionEntry(key, id, rows.getLong(5), marker, rows.getBoolean(11), metadata)); + nextKey = key; + nextVersion = id; + } + } + } + connection.commit(); + } catch (SQLException error) { throw databaseError(error); } + return new VersionPage(page, truncated ? nextKey : null, truncated ? nextVersion : null, truncated); + } + private static ListPage readListPage(ResultSet result, String bucket, String prefix, String delimiter, int maxKeys, String after) throws SQLException { List entries = new ArrayList<>(); @@ -631,15 +1167,27 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } private long lockUsage(Connection connection, String bucket) throws SQLException { + try (var statement = connection.createStatement()) { + statement.execute("SELECT pg_advisory_xact_lock(6834071092784)"); + } try (PreparedStatement query = connection.prepareStatement("SELECT used_bytes FROM cluster_usage WHERE bucket=? FOR UPDATE")) { query.setString(1, bucket); try (ResultSet result = query.executeQuery()) { - if (!result.next()) throw new SQLException("Bucket quota row is missing"); + if (!result.next()) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); return result.getLong(1); } } } + private static long occupiedBytes(Connection connection) throws SQLException { + try (var statement = connection.createStatement(); ResultSet result = statement.executeQuery( + "SELECT (SELECT COALESCE(sum(used_bytes), 0) FROM cluster_usage) + " + + "(SELECT COALESCE(sum(length), 0) FROM cluster_upload_parts)")) { + result.next(); + return result.getLong(1); + } + } + private static UUID uploadId(String id) { try { if (id == null || !id.matches("[0-9a-f-]{36}")) throw new IllegalArgumentException(); @@ -650,15 +1198,16 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } private static Upload upload(Connection connection, UUID id, String bucket, String key, boolean lock) - throws SQLException { - String sql = "SELECT bucket, object_key, content_type, created_at FROM cluster_uploads WHERE upload_id=?" + + throws SQLException, IOException { + String sql = "SELECT bucket, object_key, content_type, created_at, user_metadata, tags, acl FROM cluster_uploads WHERE upload_id=?" + (lock ? " FOR UPDATE" : ""); try (PreparedStatement query = connection.prepareStatement(sql)) { query.setObject(1, id); try (ResultSet result = query.executeQuery()) { if (!result.next() || !result.getString(1).equals(bucket) || !result.getString(2).equals(key)) throw new StoreException(404, "NoSuchUpload", "Upload not found"); - return new Upload(result.getString(3)); + return new Upload(result.getString(3), ObjectAttributes.decode(result.getBytes(5)), + ObjectAttributes.decode(result.getBytes(6)), ObjectAttributes.decode(result.getBytes(7))); } } } @@ -755,6 +1304,8 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { update.setString(6, data.etag()); update.setBytes(7, data.sha256()); update.setString(8, data.contentType()); + update.setBytes(9, ObjectAttributes.encode(data.userMetadata(), 4096)); + update.setBytes(10, ObjectAttributes.encode(data.tags(), 8192)); } private static List replicaIds(ResultSet result, int column) throws SQLException, IOException { java.sql.Array value = result.getArray(column); @@ -809,7 +1360,7 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } try (PreparedStatement query = reader.prepareStatement( "SELECT s.generation, 0, s.ordinal, s.segment_id, s.length, s.sha256, s.replica_ids, s.placement_version " + - "FROM cluster_segments s JOIN cluster_objects o ON o.generation=s.generation " + + "FROM cluster_segments s JOIN cluster_object_versions o ON o.generation=s.generation " + "UNION ALL SELECT s.upload_id, s.part_number, s.ordinal, s.segment_id, s.length, s.sha256, " + "s.replica_ids, s.placement_version FROM cluster_upload_segments s " + "ORDER BY 1, 2, 3")) { @@ -924,7 +1475,7 @@ final class ClusterStore implements ObjectStorage, MultipartStorage { } lockGc(connection, false); try (PreparedStatement referenced = connection.prepareStatement( - "SELECT EXISTS (SELECT 1 FROM cluster_segments s JOIN cluster_objects o " + + "SELECT EXISTS (SELECT 1 FROM cluster_segments s JOIN cluster_object_versions o " + "ON o.generation=s.generation WHERE s.segment_id=? AND ?=ANY(s.replica_ids) " + "UNION ALL SELECT 1 FROM cluster_upload_segments s " + "WHERE s.segment_id=? AND ?=ANY(s.replica_ids))"); diff --git a/src/cloud/lunarsky/store/Crc64Nvme.java b/src/cloud/lunarsky/store/Crc64Nvme.java new file mode 100644 index 0000000..5feb836 --- /dev/null +++ b/src/cloud/lunarsky/store/Crc64Nvme.java @@ -0,0 +1,39 @@ +package cloud.lunarsky.store; + +import java.util.zip.Checksum; +import java.util.Base64; +import java.nio.ByteBuffer; + +final class Crc64Nvme implements Checksum { + private static final long POLYNOMIAL = 0x9a6c9329ac4bc9b5L; + private static final long[] TABLE = table(); + private long state = -1L; + + private static long[] table() { + long[] values = new long[256]; + for (int index = 0; index < values.length; index++) { + long value = index; + for (int bit = 0; bit < 8; bit++) + value = (value >>> 1) ^ ((value & 1L) == 0 ? 0 : POLYNOMIAL); + values[index] = value; + } + return values; + } + + @Override public void update(int value) { + state = (state >>> 8) ^ TABLE[(int) (state ^ value) & 255]; + } + + @Override public void update(byte[] bytes, int offset, int length) { + java.util.Objects.checkFromIndexSize(offset, length, bytes.length); + for (int i = offset; i < offset + length; i++) update(bytes[i]); + } + + @Override public long getValue() { return ~state; } + + String encoded() { + return Base64.getEncoder().encodeToString(ByteBuffer.allocate(8).putLong(getValue()).array()); + } + + @Override public void reset() { state = -1L; } +} diff --git a/src/cloud/lunarsky/store/DiskStore.java b/src/cloud/lunarsky/store/DiskStore.java index bb04d72..e859ebc 100644 --- a/src/cloud/lunarsky/store/DiskStore.java +++ b/src/cloud/lunarsky/store/DiskStore.java @@ -19,23 +19,40 @@ import cloud.lunarsky.store.ObjectStorage.ListPage; final class DiskStore implements ObjectStorage { private static final long MAGIC_V1 = 0x4c534f424a303031L; private static final long MAGIC_V2 = 0x4c534f424a303032L; + private static final long MAGIC_V3 = 0x4c534f424a303033L; + private static final long MAGIC_V4 = 0x4c534f424a303034L; + private static final long MAGIC_V5 = 0x4c534f424a303035L; private static final int HEADER_V1 = 72; private static final int HEADER_V2 = 78; - private final Path root, objects, temporary; + private static final int HEADER_V3 = 82; + private static final int CHECKSUM_AREA = 512; + private static final int HEADER_V4 = HEADER_V3 + 2 + CHECKSUM_AREA; + private static final int ACL_AREA = 2048; + private static final int HEADER_V5 = HEADER_V4 + 2; + private static final int BUCKET_MAGIC = 0x4c534243; + private static final int BUCKET_MAGIC_V2 = 0x4c534244; + private static final int BUCKET_MAGIC_V3 = 0x4c534245; + private static final int VERSION_MAGIC = 0x4c53564d; + private final Path root, objects, temporary, catalog, versions; private final FileChannel lockChannel; private final FileLock processLock; private final long maxObject, maxTotal; private final Object[] locks = new Object[128]; private final NavigableMap index = new TreeMap<>(); + private final NavigableMap buckets = new TreeMap<>(); + private final NavigableMap> histories = new TreeMap<>(); private long used; private long objectCount, legacyCount; record Record(Metadata metadata, int headerLength) {} + private record VersionRecord(String id, String storageId, boolean marker, long modified) {} DiskStore(Path root, long maxObject, long maxTotal) throws IOException { this.root = root; objects = root.resolve("objects"); temporary = root.resolve("pending"); + catalog = root.resolve("buckets.bin"); + versions = root.resolve("versions"); this.maxObject = maxObject; this.maxTotal = maxTotal; Arrays.setAll(locks, i -> new Object()); @@ -49,6 +66,7 @@ final class DiskStore implements ObjectStorage { if (acquired == null) throw new IOException("Data directory is already in use"); Files.createDirectories(objects); Files.createDirectories(temporary); + Files.createDirectories(versions); syncDirectory(root); try (var paths = Files.list(temporary)) { for (Path p : paths.toList()) if (p.getFileName().toString().endsWith(".part")) Files.delete(p); @@ -70,6 +88,37 @@ final class DiskStore implements ObjectStorage { used = Math.addExact(used, meta.length()); } } + if (Files.exists(catalog)) { + try (DataInputStream input = new DataInputStream(Files.newInputStream(catalog))) { + int magic = input.readInt(); + if (magic != BUCKET_MAGIC && magic != BUCKET_MAGIC_V2 && magic != BUCKET_MAGIC_V3) + throw new IOException("Invalid bucket catalog"); + int count = input.readInt(); + if (count < 0 || count > 1000) throw new IOException("Invalid bucket catalog"); + for (int i = 0; i < count; i++) { + String name = input.readUTF(); + long created = input.readLong(); + VersioningState state = VersioningState.NEVER; + if (magic == BUCKET_MAGIC_V2 || magic == BUCKET_MAGIC_V3) { + int ordinal = input.readUnsignedByte(); + if (ordinal >= VersioningState.values().length) + throw new IOException("Invalid bucket versioning state"); + state = VersioningState.values()[ordinal]; + } + Map acl = Map.of(); + if (magic == BUCKET_MAGIC_V3) { + int size = input.readUnsignedShort(); + if (size > ACL_AREA) throw new IOException("Invalid bucket ACL"); + acl = ObjectAttributes.decode(input.readNBytes(size)); + } + if (!validBucket(name) || created < 0 || + buckets.put(name, new Bucket(name, created, state, acl)) != null) + throw new IOException("Invalid bucket catalog"); + } + if (input.read() != -1) throw new IOException("Invalid bucket catalog"); + } + } + loadHistories(); ready = true; } finally { if (!ready) { @@ -88,12 +137,251 @@ final class DiskStore implements ObjectStorage { Path root() { return root; } long maxObject() { return maxObject; } long maxTotal() { return maxTotal; } + @Override public Limits limits() { return new Limits(maxObject, maxTotal); } synchronized long usedBytes() { return used; } synchronized int indexedObjects() { return index.size(); } synchronized long objectCount() { return objectCount; } synchronized long legacyObjects() { return legacyCount; } + private static boolean validBucket(String name) { + return name.matches("[a-z0-9][a-z0-9-]{1,61}[a-z0-9]"); + } + + @Override public synchronized void ensureBucket(String bucket) throws IOException { + if (!buckets.containsKey(bucket)) createBucket(bucket); + } + + @Override public synchronized Bucket bucket(String name) { + Bucket found = buckets.get(name); + if (found == null) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + return found; + } + + @Override public synchronized List buckets() { return List.copyOf(buckets.values()); } + + @Override public synchronized void createBucket(String name) throws IOException { + if (!validBucket(name)) throw new StoreException(400, "InvalidBucketName", "Invalid bucket name"); + if (buckets.containsKey(name)) + throw new StoreException(409, "BucketAlreadyOwnedByYou", "Bucket already exists"); + if (buckets.size() >= 1000) throw new StoreException(400, "TooManyBuckets", "Bucket limit reached"); + NavigableMap next = new TreeMap<>(buckets); + next.put(name, new Bucket(name, Instant.now().toEpochMilli())); + saveBuckets(next); + buckets.clear(); + buckets.putAll(next); + } + + @Override public synchronized void deleteBucket(String name) throws IOException { + bucket(name); + if (index.values().stream().anyMatch(meta -> name.equals(meta.bucket()))) + throw new StoreException(409, "BucketNotEmpty", "Bucket contains objects"); + if (histories.entrySet().stream().anyMatch(entry -> entry.getKey().startsWith(name + "\0") && + !entry.getValue().isEmpty())) + throw new StoreException(409, "BucketNotEmpty", "Bucket contains object versions"); + if (legacyCount > 0) { + try (var paths = Files.walk(objects)) { + for (Path path : paths.filter(Files::isRegularFile).toList()) { + try (var input = new DataInputStream(Files.newInputStream(path))) { + Metadata metadata = readRecord(input).metadata(); + if (metadata.key() == null && name.equals(metadata.bucket())) + throw new StoreException(409, "BucketNotEmpty", "Bucket contains legacy objects"); + } + } + } + } + NavigableMap next = new TreeMap<>(buckets); + next.remove(name); + saveBuckets(next); + buckets.clear(); + buckets.putAll(next); + for (String key : new ArrayList<>(histories.keySet())) { + if (!key.startsWith(name + "\0") || !histories.get(key).isEmpty()) continue; + String objectKey = key.substring(name.length() + 1); + Path directory = historyDirectory(name, objectKey); + Files.deleteIfExists(directory.resolve("manifest")); + syncDirectory(directory); + histories.remove(key); + } + } + + private void saveBuckets(NavigableMap next) throws IOException { + Path pending = Files.createTempFile(temporary, "buckets-", ".part"); + try { + try (DataOutputStream output = new DataOutputStream(Files.newOutputStream(pending))) { + output.writeInt(BUCKET_MAGIC_V3); + output.writeInt(next.size()); + for (Bucket entry : next.values()) { + output.writeUTF(entry.name()); + output.writeLong(entry.created()); + output.writeByte(entry.versioning().ordinal()); + byte[] acl = ObjectAttributes.encode(entry.acl(), ACL_AREA); + output.writeShort(acl.length); + output.write(acl); + } + } + try (FileChannel channel = FileChannel.open(pending, StandardOpenOption.WRITE)) { channel.force(true); } + Files.move(pending, catalog, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + syncDirectory(root); + } finally { Files.deleteIfExists(pending); } + } + + @Override public synchronized void setVersioning(String name, VersioningState state) throws IOException { + if (state == VersioningState.NEVER) + throw new StoreException(400, "InvalidArgument", "Versioning cannot be disabled after it is enabled"); + Bucket old = bucket(name); + if (old.versioning() == VersioningState.NEVER && state == VersioningState.SUSPENDED) + throw new StoreException(400, "InvalidArgument", "Enable versioning before suspending it"); + NavigableMap next = new TreeMap<>(buckets); + next.put(name, new Bucket(name, old.created(), state, old.acl())); + saveBuckets(next); + buckets.clear(); + buckets.putAll(next); + } + + @Override public synchronized void setBucketAcl(String name, Map acl) throws IOException { + Bucket old = bucket(name); + NavigableMap next = new TreeMap<>(buckets); + next.put(name, new Bucket(name, old.created(), old.versioning(), Map.copyOf(acl))); + saveBuckets(next); + buckets.clear(); + buckets.putAll(next); + } + private static String indexKey(String bucket, String key) { return bucket + "\0" + key; } + private Path historyDirectory(String bucket, String key) { + String id = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(StandardCharsets.UTF_8))); + return versions.resolve(id.substring(0, 2)).resolve(id); + } + private Path versionPath(String bucket, String key, String storageId) { + return storageId.equals("legacy") ? objectPath(bucket, key) : + historyDirectory(bucket, key).resolve(storageId); + } + private static Metadata withVersion(Metadata old, String id, String bucket, String key) { + return new Metadata(old.length(), old.modified(), old.etag(), old.sha256(), bucket, key, + old.contentType(), old.userMetadata(), old.tags(), id, old.checksums(), old.acl()); + } + private List history(String bucket, String key) throws IOException { + List found = histories.get(indexKey(bucket, key)); + if (found != null) return found; + Metadata old = index.get(indexKey(bucket, key)); + if (old == null && Files.isRegularFile(objectPath(bucket, key))) { + try (DataInputStream input = new DataInputStream(Files.newInputStream(objectPath(bucket, key)))) { + old = readRecord(input).metadata(); + } + } + return old == null ? List.of() : List.of(new VersionRecord("null", "legacy", false, old.modified())); + } + private void saveHistory(String bucket, String key, List entries) throws IOException { + Path directory = historyDirectory(bucket, key); + if (!Files.isDirectory(directory)) { + Files.createDirectories(directory); + syncDirectory(directory.getParent()); + } + Path pending = Files.createTempFile(temporary, "history-", ".part"); + try { + try (DataOutputStream output = new DataOutputStream(Files.newOutputStream(pending))) { + output.writeInt(VERSION_MAGIC); + output.writeUTF(bucket); + output.writeUTF(key); + output.writeInt(entries.size()); + for (VersionRecord entry : entries) { + output.writeUTF(entry.id()); + output.writeUTF(entry.storageId()); + output.writeBoolean(entry.marker()); + output.writeLong(entry.modified()); + } + } + try (FileChannel channel = FileChannel.open(pending, StandardOpenOption.WRITE)) { + channel.force(true); + } + Files.move(pending, directory.resolve("manifest"), StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + syncDirectory(directory); + } finally { Files.deleteIfExists(pending); } + histories.put(indexKey(bucket, key), List.copyOf(entries)); + } + private void loadHistories() throws IOException { + Set referenced = new HashSet<>(); + try (var paths = Files.walk(versions)) { + for (Path manifest : paths.filter(p -> p.getFileName().toString().equals("manifest")).toList()) { + String bucket; + String key; + List entries = new ArrayList<>(); + try (DataInputStream input = new DataInputStream(Files.newInputStream(manifest))) { + if (input.readInt() != VERSION_MAGIC) throw new IOException("Invalid version manifest: " + manifest); + bucket = input.readUTF(); + key = input.readUTF(); + int count = input.readInt(); + if (count < 0 || count > 1_000_000 || !manifest.getParent().equals(historyDirectory(bucket, key))) + throw new IOException("Invalid version manifest: " + manifest); + Set ids = new HashSet<>(); + for (int i = 0; i < count; i++) { + String id = input.readUTF(); + String storageId = input.readUTF(); + boolean marker = input.readBoolean(); + long modified = input.readLong(); + if ((!id.equals("null") && !id.matches("[0-9a-f-]{36}")) || + (!marker && !storageId.equals("legacy") && !storageId.matches("[0-9a-f-]{36}")) || + (marker && !storageId.isEmpty()) || !ids.add(id) || modified < 0) + throw new IOException("Invalid version manifest entry: " + manifest); + entries.add(new VersionRecord(id, storageId, marker, modified)); + if (!marker) { + Path file = versionPath(bucket, key, storageId); + if (!Files.isRegularFile(file)) throw new IOException("Missing object version: " + file); + referenced.add(file); + if (!storageId.equals("legacy")) { + Record record; + try (DataInputStream data = new DataInputStream(Files.newInputStream(file))) { + record = readRecord(data); + } + Metadata meta = record.metadata(); + if (!bucket.equals(meta.bucket()) || !key.equals(meta.key()) || + Files.size(file) - record.headerLength() != meta.length()) + throw new IOException("Invalid object version: " + file); + objectCount++; + used = Math.addExact(used, meta.length()); + } + } + } + if (input.read() != -1 || histories.put(indexKey(bucket, key), List.copyOf(entries)) != null) + throw new IOException("Invalid version manifest: " + manifest); + } + if (entries.isEmpty() || entries.getFirst().marker()) index.remove(indexKey(bucket, key)); + else { + VersionRecord current = entries.getFirst(); + Path file = versionPath(bucket, key, current.storageId()); + try (DataInputStream input = new DataInputStream(Files.newInputStream(file))) { + index.put(indexKey(bucket, key), + withVersion(readRecord(input).metadata(), current.id(), bucket, key)); + } + } + } + } + try (var paths = Files.walk(versions)) { + for (Path file : paths.filter(Files::isRegularFile).toList()) { + if (!file.getFileName().toString().equals("manifest") && !referenced.contains(file)) { + Files.delete(file); + syncDirectory(file.getParent()); + } + } + } + for (var entry : histories.entrySet()) { + String[] parts = entry.getKey().split("\0", 2); + if (entry.getValue().stream().anyMatch(version -> version.storageId().equals("legacy") && !version.marker())) + continue; + Path orphan = objectPath(parts[0], parts[1]); + if (Files.isRegularFile(orphan)) { + try (DataInputStream input = new DataInputStream(Files.newInputStream(orphan))) { + Metadata old = readRecord(input).metadata(); + used -= old.length(); + if (old.key() == null) legacyCount--; + } + Files.delete(orphan); + objectCount--; + syncDirectory(orphan.getParent()); + } + } + } private Path objectPath(String bucket, String key) { String id = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(StandardCharsets.UTF_8))); return objects.resolve(id.substring(0, 2)).resolve(id); @@ -114,19 +402,24 @@ final class DiskStore implements ObjectStorage { private Object lock(Path p) { return locks[(p.hashCode() & 0x7fffffff) % locks.length]; } public Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, - String checksum, boolean createOnly, String contentType) throws IOException { + String checksum, boolean createOnly, String contentType, + Map userMetadata, Map tags, + java.util.function.Supplier> checksums, + Map acl) throws IOException { if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required"); if (length > maxObject) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit"); byte[] bucketBytes = bucket.getBytes(StandardCharsets.UTF_8); byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); byte[] typeBytes = contentType.getBytes(StandardCharsets.UTF_8); + byte[] metadataBytes = ObjectAttributes.encode(userMetadata, 4096); + byte[] tagBytes = ObjectAttributes.encode(tags, 8192); validateMetadataLengths(bucketBytes, keyBytes, typeBytes); Path destination = object(bucket, key), pending = Files.createTempFile(temporary, "upload-", ".part"); try { Metadata metadata = stagePut(pending, input, length, expectedHash, checksum, - bucket, key, contentType, bucketBytes, keyBytes, typeBytes); - installPending(destination, pending, metadata, createOnly); - return metadata; + bucket, key, contentType, bucketBytes, keyBytes, typeBytes, + metadataBytes, tagBytes, userMetadata, tags, checksums, acl); + return installPending(destination, pending, metadata, createOnly); } finally { Files.deleteIfExists(pending); } } @@ -137,9 +430,17 @@ final class DiskStore implements ObjectStorage { private Metadata stagePut(Path pending, InputStream input, long length, String expectedHash, String checksum, String bucket, String key, String contentType, - byte[] bucketBytes, byte[] keyBytes, byte[] typeBytes) throws IOException { - int headerLength = HEADER_V2 + bucketBytes.length + keyBytes.length + typeBytes.length; + byte[] bucketBytes, byte[] keyBytes, byte[] typeBytes, + byte[] metadataBytes, byte[] tagBytes, + Map userMetadata, Map tags, + java.util.function.Supplier> checksums, + Map acl) throws IOException { + byte[] aclBytes = ObjectAttributes.encode(acl, ACL_AREA); + int headerLength = HEADER_V5 + aclBytes.length + + bucketBytes.length + keyBytes.length + typeBytes.length + + metadataBytes.length + tagBytes.length; MessageDigest sha = digest("SHA-256"), md5 = digest("MD5"); + Crc64Nvme crc64 = new Crc64Nvme(); long count = 0; try (OutputStream out = Files.newOutputStream(pending)) { out.write(new byte[headerLength]); @@ -151,30 +452,47 @@ final class DiskStore implements ObjectStorage { throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size"); sha.update(buffer, 0, n); md5.update(buffer, 0, n); + crc64.update(buffer, 0, n); out.write(buffer, 0, n); } } if (count != length) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length"); byte[] hash = sha.digest(), etag = md5.digest(); - if (!MessageDigest.isEqual(hash, HexFormat.of().parseHex(expectedHash))) + if (expectedHash != null && !MessageDigest.isEqual(hash, HexFormat.of().parseHex(expectedHash))) throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch"); if (checksum != null && !Base64.getEncoder().encodeToString(hash).equals(checksum)) throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch"); + Map suppliedChecksums = checksums.get(); + Map storedChecksums = suppliedChecksums.isEmpty() ? + Map.of("x-amz-checksum-crc64nvme", crc64.encoded()) : Map.copyOf(suppliedChecksums); + byte[] checksumBytes = ObjectAttributes.encode(storedChecksums, CHECKSUM_AREA); long modified = Instant.now().toEpochMilli(); - ByteBuffer header = ByteBuffer.allocate(headerLength).putLong(MAGIC_V2).putLong(count) + ByteBuffer header = ByteBuffer.allocate(headerLength).putLong(MAGIC_V5).putLong(count) .putLong(modified).put(etag).put(hash).putShort((short) bucketBytes.length) .putShort((short) keyBytes.length).putShort((short) typeBytes.length) - .put(bucketBytes).put(keyBytes).put(typeBytes); + .putShort((short) metadataBytes.length).putShort((short) tagBytes.length) + .putShort((short) checksumBytes.length); + header.position(header.position() + CHECKSUM_AREA - checksumBytes.length); + header.put(checksumBytes).putShort((short) aclBytes.length); + header.put(aclBytes).put(bucketBytes).put(keyBytes).put(typeBytes).put(metadataBytes).put(tagBytes); header.flip(); try (FileChannel file = FileChannel.open(pending, StandardOpenOption.WRITE)) { while (header.hasRemaining()) file.write(header, header.position()); file.force(true); } - return new Metadata(count, modified, SigV4.hex(etag), hash, bucket, key, contentType); + return new Metadata(count, modified, SigV4.hex(etag), hash, bucket, key, contentType, + Map.copyOf(userMetadata), Map.copyOf(tags), null, storedChecksums, Map.copyOf(acl)); } - private void installPending(Path destination, Path pending, Metadata metadata, boolean createOnly) throws IOException { + private Metadata installPending(Path destination, Path pending, Metadata metadata, + boolean createOnly) throws IOException { synchronized (lock(destination)) { + synchronized (this) { + Bucket configured = buckets.get(metadata.bucket()); + if (configured != null && configured.versioning() != VersioningState.NEVER) + return installVersionedPending(destination, pending, metadata, createOnly, + configured.versioning()); + } long previous = 0; boolean existed = Files.exists(destination); boolean legacy = false; @@ -187,6 +505,8 @@ final class DiskStore implements ObjectStorage { } } synchronized (this) { + if (Files.exists(catalog) && !buckets.containsKey(metadata.bucket())) + throw new StoreException(404, "NoSuchBucket", "Bucket not found"); if (used - previous + metadata.length() > maxTotal) throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); Files.move(pending, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); @@ -196,16 +516,91 @@ final class DiskStore implements ObjectStorage { index.put(indexKey(metadata.bucket(), metadata.key()), metadata); syncDirectory(destination.getParent()); } + return metadata; } } + private Metadata installVersionedPending(Path destination, Path pending, Metadata metadata, + boolean createOnly, VersioningState state) throws IOException { + String bucket = metadata.bucket(); + String key = metadata.key(); + if (createOnly && index.containsKey(indexKey(bucket, key))) + throw new StoreException(412, "PreconditionFailed", "Object already exists"); + List old = history(bucket, key); + String id = state == VersioningState.ENABLED ? UUID.randomUUID().toString() : "null"; + String storageId = UUID.randomUUID().toString(); + VersionRecord discarded = null; + long replaced = 0; + if (id.equals("null")) { + for (VersionRecord entry : old) { + if (!entry.id().equals("null") || entry.marker()) continue; + discarded = entry; + try (DataInputStream input = new DataInputStream(Files.newInputStream( + versionPath(bucket, key, entry.storageId())))) { + replaced = readRecord(input).metadata().length(); + } + } + } + if (maxTotal - (used - replaced) < metadata.length()) + throw new StoreException(507, "InsufficientStorage", "Store capacity limit reached"); + Path target = versionPath(bucket, key, storageId); + Files.createDirectories(target.getParent()); + syncDirectory(target.getParent().getParent()); + Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + syncDirectory(target.getParent()); + List next = new ArrayList<>(); + next.add(new VersionRecord(id, storageId, false, metadata.modified())); + for (VersionRecord entry : old) if (!entry.id().equals(id)) next.add(entry); + saveHistory(bucket, key, next); + used += metadata.length(); + objectCount++; + Metadata current = withVersion(metadata, id, bucket, key); + index.put(indexKey(bucket, key), current); + if (discarded != null) removeStoredVersion(bucket, key, discarded, replaced); + return current; + } + + private void removeStoredVersion(String bucket, String key, VersionRecord entry, long length) throws IOException { + Path file = versionPath(bucket, key, entry.storageId()); + if (entry.storageId().equals("legacy")) { + try (DataInputStream input = new DataInputStream(Files.newInputStream(file))) { + if (readRecord(input).metadata().key() == null) legacyCount--; + } + } + Files.deleteIfExists(file); + syncDirectory(file.getParent()); + used -= length; + objectCount--; + } + public OpenObject open(String bucket, String key) throws IOException { + return open(bucket, key, null); + } + + @Override public OpenObject open(String bucket, String key, String versionId) throws IOException { Path destination = object(bucket, key); synchronized (lock(destination)) { + VersionRecord selected; + synchronized (this) { + List entries = history(bucket, key); + selected = versionId == null ? (entries.isEmpty() ? null : entries.getFirst()) : + entries.stream().filter(entry -> entry.id().equals(versionId)).findFirst().orElse(null); + } + if (selected == null) + throw new StoreException(404, versionId == null ? "NoSuchKey" : "NoSuchVersion", + "Object version not found"); + if (selected.marker()) + throw StoreException.deletedVersion(selected.id(), selected.modified(), versionId != null); final DataInputStream input; - try { input = new DataInputStream(Files.newInputStream(destination)); } + try { input = new DataInputStream(Files.newInputStream( + versionPath(bucket, key, selected.storageId()))); } catch (NoSuchFileException e) { throw new StoreException(404, "NoSuchKey", "Object not found"); } - try { return new OpenObject(readRecord(input).metadata(), input); } + try { + Bucket configured = buckets.get(bucket); + String exposedId = histories.containsKey(indexKey(bucket, key)) || + configured != null && configured.versioning() != VersioningState.NEVER ? selected.id() : null; + return new OpenObject(withVersion(readRecord(input).metadata(), exposedId, bucket, key), input); + } catch (IOException e) { input.close(); throw e; @@ -214,9 +609,19 @@ final class DiskStore implements ObjectStorage { } public void delete(String bucket, String key) throws IOException { + delete(bucket, key, null); + } + + @Override public DeleteResult delete(String bucket, String key, String versionId) throws IOException { Path destination = object(bucket, key); synchronized (lock(destination)) { - if (!Files.exists(destination)) return; + synchronized (this) { + Bucket configured = buckets.get(bucket); + VersioningState state = configured == null ? VersioningState.NEVER : configured.versioning(); + if (state != VersioningState.NEVER || versionId != null) + return deleteVersioned(bucket, key, versionId, state); + } + if (!Files.exists(destination)) return new DeleteResult(null, false); long length; boolean legacy; try (var input = new DataInputStream(Files.newInputStream(destination))) { @@ -232,9 +637,146 @@ final class DiskStore implements ObjectStorage { index.remove(indexKey(bucket, key)); syncDirectory(destination.getParent()); } + return new DeleteResult(null, false); } } + private DeleteResult deleteVersioned(String bucket, String key, String versionId, + VersioningState state) throws IOException { + List old = history(bucket, key); + List next = new ArrayList<>(); + VersionRecord removed = null; + if (versionId != null) { + for (VersionRecord entry : old) { + if (entry.id().equals(versionId)) removed = entry; + else next.add(entry); + } + if (removed == null) + throw new StoreException(404, "NoSuchVersion", "Object version not found"); + } else { + String id = state == VersioningState.ENABLED ? UUID.randomUUID().toString() : "null"; + next.add(new VersionRecord(id, "", true, Instant.now().toEpochMilli())); + for (VersionRecord entry : old) { + if (state == VersioningState.SUSPENDED && entry.id().equals("null")) removed = entry; + else next.add(entry); + } + } + saveHistory(bucket, key, next); + updateCurrentIndex(bucket, key, next); + if (removed != null && !removed.marker()) { + Path file = versionPath(bucket, key, removed.storageId()); + long length; + try (DataInputStream input = new DataInputStream(Files.newInputStream(file))) { + length = readRecord(input).metadata().length(); + } + removeStoredVersion(bucket, key, removed, length); + } + if (versionId != null) return new DeleteResult(versionId, removed.marker()); + return new DeleteResult(next.getFirst().id(), true); + } + + private void updateCurrentIndex(String bucket, String key, List entries) throws IOException { + if (entries.isEmpty() || entries.getFirst().marker()) { + index.remove(indexKey(bucket, key)); + return; + } + VersionRecord latest = entries.getFirst(); + try (DataInputStream input = new DataInputStream(Files.newInputStream( + versionPath(bucket, key, latest.storageId())))) { + index.put(indexKey(bucket, key), + withVersion(readRecord(input).metadata(), latest.id(), bucket, key)); + } + } + + @Override public Map tags(String bucket, String key) throws IOException { + return tags(bucket, key, null); + } + + @Override public Map tags(String bucket, String key, String versionId) throws IOException { + try (OpenObject object = open(bucket, key, versionId)) { return object.metadata().tags(); } + } + + @Override public void setTags(String bucket, String key, Map tags) throws IOException { + setTags(bucket, key, null, tags); + } + + @Override public void setTags(String bucket, String key, String versionId, + Map tags) throws IOException { + rewriteAttributes(bucket, key, versionId, tags, null); + } + + @Override public void setObjectAcl(String bucket, String key, String versionId, + Map acl) throws IOException { + rewriteAttributes(bucket, key, versionId, null, acl); + } + + private void rewriteAttributes(String bucket, String key, String versionId, + Map tags, Map acl) throws IOException { + Path destination = object(bucket, key); + synchronized (lock(destination)) { + VersionRecord selected; + boolean current; + synchronized (this) { + List entries = history(bucket, key); + selected = versionId == null ? (entries.isEmpty() ? null : entries.getFirst()) : + entries.stream().filter(entry -> entry.id().equals(versionId)).findFirst().orElse(null); + current = selected != null && !entries.isEmpty() && selected == entries.getFirst(); + } + if (selected == null || selected.marker()) + throw new StoreException(404, versionId == null ? "NoSuchKey" : "NoSuchVersion", + "Object version not found"); + destination = versionPath(bucket, key, selected.storageId()); + Path pending = Files.createTempFile(temporary, "tags-", ".part"); + try { + try (DataInputStream input = new DataInputStream(Files.newInputStream(destination)); + OutputStream output = Files.newOutputStream(pending)) { + Metadata old = readRecord(input).metadata(); + if (old.key() == null) throw new StoreException(501, "NotImplemented", "Legacy object tags are unsupported"); + Metadata updated = new Metadata(old.length(), old.modified(), old.etag(), old.sha256(), + bucket, key, old.contentType(), old.userMetadata(), + tags == null ? old.tags() : Map.copyOf(tags), + old.versionId(), old.checksums(), + acl == null ? old.acl() : Map.copyOf(acl)); + output.write(recordHeader(updated)); + if (input.transferTo(output) != old.length()) throw new IOException("Object length changed during tag update"); + } + try (FileChannel channel = FileChannel.open(pending, StandardOpenOption.WRITE)) { channel.force(true); } + synchronized (this) { + Files.move(pending, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + if (current) { + Metadata old = index.get(indexKey(bucket, key)); + index.put(indexKey(bucket, key), new Metadata(old.length(), old.modified(), old.etag(), + old.sha256(), bucket, key, old.contentType(), old.userMetadata(), + tags == null ? old.tags() : Map.copyOf(tags), + old.versionId(), old.checksums(), + acl == null ? old.acl() : Map.copyOf(acl))); + } + syncDirectory(destination.getParent()); + } + } finally { Files.deleteIfExists(pending); } + } + } + + private static byte[] recordHeader(Metadata metadata) { + byte[] bucket = metadata.bucket().getBytes(StandardCharsets.UTF_8); + byte[] key = metadata.key().getBytes(StandardCharsets.UTF_8); + byte[] type = metadata.contentType().getBytes(StandardCharsets.UTF_8); + byte[] custom = ObjectAttributes.encode(metadata.userMetadata(), 4096); + byte[] tags = ObjectAttributes.encode(metadata.tags(), 8192); + byte[] checksums = ObjectAttributes.encode(metadata.checksums(), CHECKSUM_AREA); + byte[] acl = ObjectAttributes.encode(metadata.acl(), ACL_AREA); + ByteBuffer header = ByteBuffer.allocate(HEADER_V5 + acl.length + + bucket.length + key.length + type.length + custom.length + tags.length) + .putLong(MAGIC_V5).putLong(metadata.length()).putLong(metadata.modified()) + .put(HexFormat.of().parseHex(metadata.etag())).put(metadata.sha256()) + .putShort((short) bucket.length).putShort((short) key.length).putShort((short) type.length) + .putShort((short) custom.length).putShort((short) tags.length) + .putShort((short) checksums.length); + header.position(header.position() + CHECKSUM_AREA - checksums.length); + header.put(checksums).putShort((short) acl.length); + return header.put(acl).put(bucket).put(key).put(type).put(custom).put(tags).array(); + } + public synchronized ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) { List entries = new ArrayList<>(); List prefixes = new ArrayList<>(); @@ -271,9 +813,58 @@ final class DiskStore implements ObjectStorage { return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated); } + @Override public synchronized VersionPage listVersions(String bucket, String prefix, + String keyMarker, String versionMarker, + int maxKeys) throws IOException { + bucket(bucket); + if (versionMarker != null && keyMarker == null) + throw new StoreException(400, "InvalidArgument", "Version marker requires a key marker"); + if (maxKeys == 0) return new VersionPage(List.of(), null, null, false); + NavigableSet keys = new TreeSet<>(); + for (String name : histories.keySet()) if (name.startsWith(bucket + "\0")) keys.add(name.substring(bucket.length() + 1)); + for (Metadata meta : index.values()) if (bucket.equals(meta.bucket())) keys.add(meta.key()); + List page = new ArrayList<>(); + String nextKey = null; + String nextVersion = null; + boolean truncated = false; + for (String key : keys) { + if (!key.startsWith(prefix) || keyMarker != null && key.compareTo(keyMarker) < 0) continue; + List entries = history(bucket, key); + boolean pastMarker = keyMarker == null || !key.equals(keyMarker) || versionMarker == null; + for (int i = 0; i < entries.size(); i++) { + VersionRecord entry = entries.get(i); + if (keyMarker != null && key.equals(keyMarker)) { + if (versionMarker == null) continue; + if (!pastMarker) { + if (entry.id().equals(versionMarker)) pastMarker = true; + continue; + } + } + if (page.size() == maxKeys) { + truncated = true; + break; + } + Metadata meta = null; + if (!entry.marker()) { + try (DataInputStream input = new DataInputStream(Files.newInputStream( + versionPath(bucket, key, entry.storageId())))) { + meta = withVersion(readRecord(input).metadata(), entry.id(), bucket, key); + } + } + page.add(new VersionEntry(key, entry.id(), entry.modified(), entry.marker(), i == 0, meta)); + nextKey = key; + nextVersion = entry.id(); + } + if (truncated) break; + } + return new VersionPage(page, truncated ? nextKey : null, truncated ? nextVersion : null, truncated); + } + static Record readRecord(DataInputStream in) throws IOException { long magic = in.readLong(); - if (magic != MAGIC_V1 && magic != MAGIC_V2) throw new IOException("Invalid object record"); + if (magic != MAGIC_V1 && magic != MAGIC_V2 && magic != MAGIC_V3 && magic != MAGIC_V4 && + magic != MAGIC_V5) + throw new IOException("Invalid object record"); long length = in.readLong(), modified = in.readLong(); byte[] md5 = new byte[16], sha = new byte[32]; in.readFully(md5); @@ -283,8 +874,30 @@ final class DiskStore implements ObjectStorage { return new Record(new Metadata(length, modified, SigV4.hex(md5), sha, null, null, "application/octet-stream"), HEADER_V1); int bucketLength = in.readUnsignedShort(), keyLength = in.readUnsignedShort(), typeLength = in.readUnsignedShort(); + int metadataLength = magic == MAGIC_V3 || magic == MAGIC_V4 || magic == MAGIC_V5 ? in.readUnsignedShort() : 0; + int tagsLength = magic == MAGIC_V3 || magic == MAGIC_V4 || magic == MAGIC_V5 ? in.readUnsignedShort() : 0; + int checksumLength = magic == MAGIC_V4 || magic == MAGIC_V5 ? in.readUnsignedShort() : 0; if (bucketLength < 1 || bucketLength > 63 || keyLength < 1 || keyLength > 1024 || typeLength < 1 || typeLength > 255) throw new IOException("Invalid object record metadata"); + if (metadataLength > 4096 || tagsLength > 8192 || checksumLength > CHECKSUM_AREA) + throw new IOException("Invalid object attributes"); + Map checksums = Map.of(); + if (magic == MAGIC_V4 || magic == MAGIC_V5) { + byte[] area = in.readNBytes(CHECKSUM_AREA); + if (area.length != CHECKSUM_AREA) throw new IOException("Truncated checksum attributes"); + checksums = ObjectAttributes.decode(Arrays.copyOfRange(area, + CHECKSUM_AREA - checksumLength, CHECKSUM_AREA)); + } + Map acl = Map.of(); + int aclLength = 0; + if (magic == MAGIC_V5) { + int size = in.readUnsignedShort(); + if (size > ACL_AREA) throw new IOException("Invalid object ACL"); + byte[] bytes = in.readNBytes(size); + if (bytes.length != size) throw new IOException("Truncated object ACL"); + acl = ObjectAttributes.decode(bytes); + aclLength = size; + } String bucket = utf8(in.readNBytes(bucketLength)); String key = utf8(in.readNBytes(keyLength)); String contentType = utf8(in.readNBytes(typeLength)); @@ -292,8 +905,13 @@ final class DiskStore implements ObjectStorage { key.getBytes(StandardCharsets.UTF_8).length != keyLength || contentType.getBytes(StandardCharsets.UTF_8).length != typeLength) throw new IOException("Invalid object record metadata"); + Map metadata = ObjectAttributes.decode(in.readNBytes(metadataLength)); + Map tags = ObjectAttributes.decode(in.readNBytes(tagsLength)); return new Record(new Metadata(length, modified, SigV4.hex(md5), sha, - bucket, key, contentType), HEADER_V2 + bucketLength + keyLength + typeLength); + bucket, key, contentType, metadata, tags, null, checksums, acl), + (magic == MAGIC_V5 ? HEADER_V5 + aclLength : + magic == MAGIC_V4 ? HEADER_V4 : magic == MAGIC_V3 ? HEADER_V3 : HEADER_V2) + + bucketLength + keyLength + typeLength + metadataLength + tagsLength); } private static String utf8(byte[] bytes) throws IOException { return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) diff --git a/src/cloud/lunarsky/store/Main.java b/src/cloud/lunarsky/store/Main.java index 2050cf3..1f6080f 100644 --- a/src/cloud/lunarsky/store/Main.java +++ b/src/cloud/lunarsky/store/Main.java @@ -19,53 +19,128 @@ import java.util.UUID; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.security.MessageDigest; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; public final class Main { + private static final String CAPABILITIES_PATH = "/_objectstore/capabilities"; private final ObjectStorage store; private final SigV4 authentication; + private final String owner; private final String bucket; private final MultipartStorage multipart; + private final ClientLimits clientLimits; private final Semaphore slots = new Semaphore(16); Main(DiskStore store, SigV4 authentication, String bucket) throws IOException { this(store, new MultipartStore(store), authentication, bucket); } - Main(ObjectStorage store, MultipartStorage multipart, SigV4 authentication, String bucket) { + Main(ObjectStorage store, MultipartStorage multipart, SigV4 authentication, String bucket) throws IOException { + this(store, multipart, authentication, bucket, ClientLimits.disabled()); + } + + Main(ObjectStorage store, MultipartStorage multipart, SigV4 authentication, String bucket, + ClientLimits clientLimits) throws IOException { this.store = store; this.multipart = multipart; this.authentication = authentication; + this.owner = authentication.root(); this.bucket = bucket; + this.clientLimits = clientLimits; + store.ensureBucket(bucket); } void handle(HttpExchange exchange) throws IOException { - boolean admitted = slots.tryAcquire(); + boolean admitted = false; + ClientLimits.Client client = null; String requestId = UUID.randomUUID().toString(); exchange.getResponseHeaders().set("x-amz-request-id", requestId); exchange.getResponseHeaders().set("X-Content-Type-Options", "nosniff"); try { + client = clientLimits.enter(exchange); + admitted = slots.tryAcquire(); if (!admitted) throw new StoreException(503, "SlowDown", "Too many concurrent requests"); if (handleStatus(exchange)) return; - String hash = authentication.verify(exchange.getRequestMethod(), exchange.getRequestURI(), exchange.getRequestHeaders()); + SigV4.Verified verified = anonymousRead(exchange) + ? new SigV4.Verified("UNSIGNED-PAYLOAD", exchange.getRequestURI().getRawQuery(), + null, null, null, null, null) + : authentication.verifyRequest(exchange.getRequestMethod(), + exchange.getRequestURI(), exchange.getRequestHeaders()); + String hash = verified.payload(); + String principal = verified.principal(); String path = SigV4.decode(exchange.getRequestURI().getRawPath()); - Map query = query(exchange.getRequestURI().getRawQuery()); - if (path.equals("/" + bucket) || path.equals("/" + bucket + "/")) { - handleBucket(exchange, query, hash); + Map query = query(verified.applicationQuery()); + if (path.equals(CAPABILITIES_PATH)) { + requireOwner(principal); + if (!exchange.getRequestMethod().equals("GET")) unsupported("Capability operation"); + if (!query.isEmpty()) + throw new StoreException(400, "InvalidArgument", "Capability request has unsupported query parameters"); + requireEmptyBody(exchange, hash); + capabilities(exchange); + } else if (path.equals("/")) { + requireOwner(principal); + if (!exchange.getRequestMethod().equals("GET") || + !(query.isEmpty() || query.size() == 1 && "ListBuckets".equals(query.get("x-id")))) + unsupported("Service operation"); + requireEmptyBody(exchange, hash); + listBuckets(exchange); } else { - handleObject(exchange, path, query, hash); + int slash = path.indexOf('/', 1); + String requestedBucket = slash < 0 ? path.substring(1) : path.substring(1, slash); + if (requestedBucket.isEmpty()) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + if (slash < 0 || slash == path.length() - 1) { + handleBucket(exchange, query, hash, requestedBucket, principal); + } else { + store.bucket(requestedBucket); + handleObject(exchange, path.substring(slash + 1), query, verified, requestedBucket); + } } - } catch (StoreException error) { sendError(exchange, error.status, error.code, error.getMessage(), requestId); } + } catch (StoreException error) { + if (error.status == 503 && error.code.equals("SlowDown")) + exchange.getResponseHeaders().set("Retry-After", "1"); + if (anonymousRead(exchange) && error.status == 404) + error = new StoreException(403, "AccessDenied", "Access denied"); + if (error.deleteMarker) { + exchange.getResponseHeaders().set("x-amz-delete-marker", "true"); + exchange.getResponseHeaders().set("x-amz-version-id", error.versionId); + if (error.modified >= 0) exchange.getResponseHeaders().set("Last-Modified", + DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) + .format(Instant.ofEpochMilli(error.modified))); + } + sendError(exchange, error.status, error.code, error.getMessage(), requestId); + } catch (Exception error) { System.err.println("ObjectStore request failed: " + requestId + " " + error.getClass().getSimpleName()); sendError(exchange, 500, "InternalError", "Storage operation failed", requestId); } finally { - if (admitted) slots.release(); - exchange.close(); + try { exchange.close(); } + finally { + if (admitted) slots.release(); + clientLimits.leave(client); + } } } + private static boolean anonymousRead(HttpExchange exchange) { + if (!exchange.getRequestMethod().equals("GET") && + !exchange.getRequestMethod().equals("HEAD")) return false; + if (exchange.getRequestHeaders().containsKey("authorization")) return false; + if (exchange.getRequestHeaders().keySet().stream().anyMatch( + name -> name.toLowerCase(Locale.ROOT).startsWith("x-amz-"))) return false; + String query = exchange.getRequestURI().getRawQuery(); + return query == null || !query.toLowerCase(Locale.ROOT).contains("x-amz-"); + } + + private void requireOwner(String principal) { + if (!owner.equals(principal)) throw new StoreException(403, "AccessDenied", "Access denied"); + } + + private void requireBucket(String bucket, String principal, int permission) throws IOException { + Acl.require(store.bucket(bucket).acl(), principal, owner, permission); + } + private boolean handleStatus(HttpExchange exchange) throws IOException { if (!exchange.getRequestMethod().equals("GET")) return false; String path = exchange.getRequestURI().getRawPath(); @@ -85,15 +160,145 @@ public final class Main { return true; } - private void handleBucket(HttpExchange exchange, Map query, String hash) throws IOException { + private void capabilities(HttpExchange exchange) throws IOException { + ObjectStorage.Limits limits = store.limits(); + String mode = store instanceof ClusterStore ? "cluster" : "disk"; + String body = """ + { + "schemaVersion": 1, + "service": "lunarsky-objectstore", + "serviceVersion": "%s", + "storageMode": "%s", + "operations": [ + "ListBuckets", "CreateBucket", "HeadBucket", "DeleteBucket", "ListObjectsV2", + "PutObject", "GetObject", "HeadObject", "DeleteObject", "CopyObject", + "GetObjectTagging", "PutObjectTagging", "DeleteObjectTagging", + "CreateMultipartUpload", "UploadPart", "ListParts", "CompleteMultipartUpload", + "AbortMultipartUpload", "ListMultipartUploads", + "GetBucketVersioning", "PutBucketVersioning", "ListObjectVersions", + "GetBucketAcl", "PutBucketAcl", "GetObjectAcl", "PutObjectAcl" + ], + "limits": { + "maxObjectBytes": %d, + "maxTotalBytes": %d, + "maxParts": 10000 + } + } + """.formatted(Version.VALUE, mode, limits.maxObjectBytes(), limits.maxTotalBytes()); + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json; charset=utf-8"); + exchange.getResponseHeaders().set("Cache-Control", "no-store"); + exchange.sendResponseHeaders(200, bytes.length); + exchange.getResponseBody().write(bytes); + } + + private void listBuckets(HttpExchange exchange) throws IOException { + StringBuilder body = new StringBuilder("" + + "" + owner + "ObjectStore"); + for (ObjectStorage.Bucket entry : store.buckets()) + body.append("").append(xml(entry.name())).append("") + .append(Instant.ofEpochMilli(entry.created())).append(""); + sendXml(exchange, 200, body.append("").toString()); + } + + private void handleBucket(HttpExchange exchange, Map query, String hash, + String bucketName, String principal) throws IOException { + String method = exchange.getRequestMethod(); + if (query.containsKey("acl") && query.get("acl").isEmpty() && + query.keySet().stream().allMatch(java.util.Set.of("acl", "x-id")::contains) && + (!query.containsKey("x-id") || + query.get("x-id").equals(method.equals("GET") ? "GetBucketAcl" : "PutBucketAcl")) && + (method.equals("GET") || method.equals("PUT"))) { + var current = store.bucket(bucketName); + Acl.require(current.acl(), principal, owner, + method.equals("GET") ? Acl.READ_ACP : Acl.WRITE_ACP); + if (method.equals("GET")) { + requireEmptyBody(exchange, hash); + sendXml(exchange, 200, Acl.xml(current.acl(), owner)); + } else { + if (exchange.getRequestHeaders().containsKey("x-amz-acl") || + exchange.getRequestHeaders().keySet().stream().anyMatch( + name -> name.toLowerCase(Locale.ROOT).startsWith("x-amz-grant-"))) { + requireEmptyBody(exchange, hash); + verifyContentMd5(exchange.getRequestHeaders(), new byte[0]); + store.setBucketAcl(bucketName, Acl.fromHeaders(exchange.getRequestHeaders(), + authentication.identities())); + } else { + byte[] body = signedBody(exchange, hash, 65536); + verifyContentMd5(exchange.getRequestHeaders(), body); + store.setBucketAcl(bucketName, Acl.fromXml(body, + owner, authentication.identities())); + } + exchange.sendResponseHeaders(200, -1); + } + return; + } + String bucketOperation = switch (method) { + case "PUT" -> "CreateBucket"; + case "HEAD" -> "HeadBucket"; + case "DELETE" -> "DeleteBucket"; + default -> null; + }; + if (bucketOperation != null && (query.isEmpty() || + query.size() == 1 && bucketOperation.equals(query.get("x-id")))) { + requireEmptyBody(exchange, hash); + switch (method) { + case "PUT" -> { + requireOwner(principal); + Map acl = Acl.fromHeaders(exchange.getRequestHeaders(), + authentication.identities()); + store.createBucket(bucketName); + if (!acl.isEmpty()) store.setBucketAcl(bucketName, acl); + exchange.getResponseHeaders().set("Location", "/" + bucketName); + exchange.sendResponseHeaders(200, -1); + } + case "HEAD" -> { + requireBucket(bucketName, principal, Acl.READ); + exchange.sendResponseHeaders(200, -1); + } + case "DELETE" -> { + requireOwner(principal); + if (bucketName.equals(bucket)) + throw new StoreException(409, "BucketNotEmpty", "The configured default bucket cannot be deleted"); + synchronized (multipart) { + if (!multipart.listUploads(bucketName, "").isEmpty()) + throw new StoreException(409, "BucketNotEmpty", "Bucket has active uploads"); + store.deleteBucket(bucketName); + } + exchange.sendResponseHeaders(204, -1); + } + } + return; + } + store.bucket(bucketName); + String versioningOperation = method.equals("GET") ? "GetBucketVersioning" : + method.equals("PUT") ? "PutBucketVersioning" : null; + if (query.containsKey("versioning") && query.get("versioning").isEmpty() && + (query.size() == 1 || query.size() == 2 && versioningOperation != null && + versioningOperation.equals(query.get("x-id")))) { + requireOwner(principal); + handleVersioning(exchange, hash, bucketName); + return; + } + if (method.equals("GET") && query.containsKey("versions") && query.get("versions").isEmpty()) { + requireOwner(principal); + if (!query.keySet().stream().allMatch(java.util.Set.of("versions", "prefix", "key-marker", + "version-id-marker", "max-keys", "encoding-type", "x-id")::contains) || + query.containsKey("x-id") && !"ListObjectVersions".equals(query.get("x-id"))) + unsupported("Version listing option"); + requireEmptyBody(exchange, hash); + listVersions(exchange, query, bucketName); + return; + } if (exchange.getRequestMethod().equals("GET") && query.containsKey("uploads")) { + requireOwner(principal); if (!query.get("uploads").isEmpty() || !query.keySet().stream().allMatch(java.util.Set.of("uploads", "prefix", "key-marker", "upload-id-marker", "max-uploads", "x-id")::contains) || (query.containsKey("x-id") && !"ListMultipartUploads".equals(query.get("x-id")))) unsupported("Bucket operation"); requireEmptyBody(exchange, hash); - listUploads(exchange, query); + listUploads(exchange, query, bucketName); return; } if (!exchange.getRequestMethod().equals("GET") || !"2".equals(query.get("list-type")) || @@ -101,43 +306,108 @@ public final class Main { "continuation-token", "start-after", "encoding-type", "x-id")::contains) || (query.containsKey("x-id") && !"ListObjectsV2".equals(query.get("x-id")))) unsupported("Bucket operation"); + requireBucket(bucketName, principal, Acl.READ); requireEmptyBody(exchange, hash); - listObjects(exchange, query); + listObjects(exchange, query, bucketName); } - private void handleObject(HttpExchange exchange, String path, Map query, String hash) throws IOException { - String prefix = "/" + bucket + "/"; - if (!path.startsWith(prefix)) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); - String key = path.substring(prefix.length()); + private void handleObject(HttpExchange exchange, String key, Map query, + SigV4.Verified verified, String bucketName) throws IOException { + String hash = verified.payload(); + String principal = verified.principal(); if (key.isEmpty() || key.getBytes(StandardCharsets.UTF_8).length > 1024 || key.indexOf('\0') >= 0) throw new StoreException(400, "InvalidArgument", "Invalid object key"); String method = exchange.getRequestMethod(); + if (query.containsKey("acl") && query.get("acl").isEmpty() && + query.keySet().stream().allMatch(java.util.Set.of("acl", "versionId", "x-id")::contains) && + (!query.containsKey("x-id") || + query.get("x-id").equals(method.equals("GET") ? "GetObjectAcl" : "PutObjectAcl")) && + (method.equals("GET") || method.equals("PUT"))) { + String versionId = query.get("versionId"); + try (var object = store.open(bucketName, key, versionId)) { + var metadata = object.metadata(); + if (method.equals("GET")) { + Acl.require(metadata.acl(), principal, owner, Acl.READ_ACP); + requireEmptyBody(exchange, hash); + sendXml(exchange, 200, Acl.xml(metadata.acl(), owner)); + } else { + if (!owner.equals(principal)) { + if (metadata.versionId() == null || metadata.versionId().equals("null")) + throw new StoreException(403, "AccessDenied", "Access denied"); + Acl.require(metadata.acl(), principal, owner, Acl.WRITE_ACP); + } + Map acl; + if (exchange.getRequestHeaders().containsKey("x-amz-acl") || + exchange.getRequestHeaders().keySet().stream().anyMatch( + name -> name.toLowerCase(Locale.ROOT).startsWith("x-amz-grant-"))) { + requireEmptyBody(exchange, hash); + verifyContentMd5(exchange.getRequestHeaders(), new byte[0]); + acl = Acl.fromHeaders(exchange.getRequestHeaders(), authentication.identities()); + } else { + byte[] body = signedBody(exchange, hash, 65536); + verifyContentMd5(exchange.getRequestHeaders(), body); + acl = Acl.fromXml(body, owner, authentication.identities()); + } + store.setObjectAcl(bucketName, key, metadata.versionId(), acl); + exchange.sendResponseHeaders(200, -1); + } + } + return; + } + String taggingOperation = switch (method) { + case "GET" -> "GetObjectTagging"; + case "PUT" -> "PutObjectTagging"; + case "DELETE" -> "DeleteObjectTagging"; + default -> null; + }; + if (taggingOperation != null && query.containsKey("tagging") && query.get("tagging").isEmpty() && + query.keySet().stream().allMatch(java.util.Set.of("tagging", "versionId", "x-id")::contains) && + (!query.containsKey("x-id") || taggingOperation.equals(query.get("x-id")))) { + validateObjectHeaders(exchange.getRequestHeaders()); + requireOwner(principal); + handleTagging(exchange, method, key, hash, bucketName, query.get("versionId")); + return; + } boolean multipartRequest = multipartRequest(method, query); - if (!multipartRequest && !query.isEmpty() && !(query.size() == 1 && + boolean versionRequest = (method.equals("GET") || method.equals("HEAD") || method.equals("DELETE")) && + query.containsKey("versionId") && (query.size() == 1 || query.size() == 2 && query.containsKey("x-id")); + if (versionRequest && query.containsKey("x-id") && + !(method.equals("GET") ? "GetObject" : method.equals("HEAD") ? "HeadObject" : "DeleteObject") + .equals(query.get("x-id"))) unsupported("Version query operation"); + if (!multipartRequest && !versionRequest && !query.isEmpty() && !(query.size() == 1 && ("PutObject".equals(query.get("x-id")) || "CopyObject".equals(query.get("x-id")) || "GetObject".equals(query.get("x-id")) || "HeadObject".equals(query.get("x-id")) || "DeleteObject".equals(query.get("x-id"))))) unsupported("Query operation"); validateObjectHeaders(exchange.getRequestHeaders()); if (multipartRequest) { - handleMultipart(exchange, method, query, key, hash); + if (method.equals("GET")) requireOwner(principal); + else requireBucket(bucketName, principal, Acl.WRITE); + handleMultipart(exchange, method, query, key, verified, bucketName); return; } boolean copy = exchange.getRequestHeaders().containsKey("x-amz-copy-source"); if (!method.equals("PUT") && (copy || exchange.getRequestHeaders().containsKey("content-md5") || exchange.getRequestHeaders().containsKey("x-amz-metadata-directive") || + exchange.getRequestHeaders().containsKey("x-amz-tagging") || + exchange.getRequestHeaders().containsKey("x-amz-tagging-directive") || exchange.getRequestHeaders().containsKey("x-amz-sdk-checksum-algorithm") || exchange.getRequestHeaders().keySet().stream().anyMatch(name -> - name.toLowerCase(Locale.ROOT).startsWith("x-amz-checksum-")))) + name.toLowerCase(Locale.ROOT).startsWith("x-amz-checksum-") && + !name.equalsIgnoreCase("x-amz-checksum-mode")))) unsupported("Object upload header"); if (!method.equals("PUT") || copy) requireEmptyBody(exchange, hash); switch (method) { case "PUT" -> { - if (copy) copyObject(exchange, key); - else putObject(exchange, key, hash); + requireBucket(bucketName, principal, Acl.WRITE); + if (copy) copyObject(exchange, key, bucketName, principal); + else putObject(exchange, key, verified, bucketName); + } + case "GET", "HEAD" -> readObject(exchange, key, bucketName, query.get("versionId"), principal); + case "DELETE" -> { + requireBucket(bucketName, principal, Acl.WRITE); + deleteObject(exchange, key, bucketName, query.get("versionId")); } - case "GET", "HEAD" -> readObject(exchange, key); - case "DELETE" -> deleteObject(exchange, key); default -> unsupported("HTTP method"); } } @@ -145,37 +415,59 @@ public final class Main { private static void validateObjectHeaders(com.sun.net.httpserver.Headers headers) { for (String name : headers.keySet()) { String lower = name.toLowerCase(java.util.Locale.ROOT); - if (lower.startsWith("x-amz-") && !java.util.Set.of("x-amz-date", "x-amz-content-sha256", + if (lower.startsWith("x-amz-") && !lower.startsWith("x-amz-meta-") && !java.util.Set.of("x-amz-date", "x-amz-content-sha256", "x-amz-sdk-checksum-algorithm", "x-amz-user-agent", "x-amz-copy-source", - "x-amz-metadata-directive").contains(lower) && !lower.startsWith("x-amz-checksum-")) + "x-amz-metadata-directive", "x-amz-tagging", "x-amz-tagging-directive", + "x-amz-decoded-content-length", "x-amz-trailer", "x-amz-checksum-mode", + "x-amz-acl").contains(lower) && !lower.startsWith("x-amz-checksum-") && + !lower.startsWith("x-amz-grant-")) unsupported("Amazon header"); - if (lower.startsWith("x-amz-meta-") || lower.startsWith("x-amz-server-side-") || - lower.startsWith("x-amz-acl") || - lower.startsWith("x-amz-grant") || lower.startsWith("x-amz-tagging") || + if (lower.startsWith("x-amz-server-side-") || lower.startsWith("x-amz-copy-source-")) unsupported("Object metadata, encryption, ACL or tagging header"); } } - private void putObject(HttpExchange exchange, String key, String hash) throws IOException { + private void putObject(HttpExchange exchange, String key, SigV4.Verified verified, + String bucketName) throws IOException { var headers = exchange.getRequestHeaders(); String length = SigV4.single(headers, "content-length"), condition = SigV4.single(headers, "if-none-match"); if (condition != null && !condition.equals("*")) unsupported("Write condition"); long bytes; try { bytes = length == null ? -1 : Long.parseLong(length); } catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid Content-Length"); } - if (headers.containsKey("content-encoding")) unsupported("Encoded payload"); + if (!verified.streaming() && headers.containsKey("content-encoding")) unsupported("Encoded payload"); if (headers.containsKey("x-amz-metadata-directive")) unsupported("Copy metadata directive"); + if (headers.containsKey("x-amz-tagging-directive")) unsupported("Copy tagging directive"); UploadChecksums checksums = UploadChecksums.from(headers); + AwsChunkedInputStream chunked = verified.streaming() ? chunkedBody(exchange, verified) : null; + if (chunked != null) bytes = decodedLength(headers); + else if (headers.containsKey("x-amz-decoded-content-length") || headers.containsKey("x-amz-trailer")) + unsupported("Chunked upload header"); String type = contentType(headers); - ObjectStorage.Metadata data = store.put(bucket, key, checksums.verifying(exchange.getRequestBody()), - bytes, hash, checksums.sha256(), condition != null, type); + Map metadata = ObjectAttributes.userMetadata(headers); + Map tags = ObjectAttributes.tagsHeader(SigV4.single(headers, "x-amz-tagging")); + Map acl = Acl.fromHeaders(headers, authentication.identities()); + if (!acl.isEmpty() && !owner.equals(verified.principal())) + requireBucket(bucketName, verified.principal(), Acl.WRITE_ACP); + ObjectStorage.Metadata data = store.put(bucketName, key, + checksums.verifying(chunked == null ? exchange.getRequestBody() : chunked), + bytes, expectedPayloadHash(verified.payload()), checksums.sha256(), condition != null, type, + metadata, tags, () -> { + if (chunked == null || chunked.trailerValue() == null) return checksums.metadata(); + String trailer = SigV4.single(headers, "x-amz-trailer"); + if (!checksums.metadata().isEmpty()) + throw new StoreException(400, "InvalidRequest", "Supply one checksum algorithm"); + return Map.of(trailer, chunked.trailerValue()); + }, acl); exchange.getResponseHeaders().set("ETag", "\"" + data.etag() + "\""); + if (data.versionId() != null) exchange.getResponseHeaders().set("x-amz-version-id", data.versionId()); exchange.getResponseHeaders().set("x-amz-checksum-sha256", Base64.getEncoder().encodeToString(data.sha256())); - checksums.response(exchange.getResponseHeaders()); + data.checksums().forEach(exchange.getResponseHeaders()::set); exchange.sendResponseHeaders(200, -1); } - private void copyObject(HttpExchange exchange, String key) throws IOException { + private void copyObject(HttpExchange exchange, String key, String bucketName, + String principal) throws IOException { var headers = exchange.getRequestHeaders(); if (headers.containsKey("content-encoding") || headers.containsKey("content-md5") || headers.containsKey("if-none-match") || headers.containsKey("x-amz-sdk-checksum-algorithm") || @@ -184,12 +476,21 @@ public final class Main { String source = SigV4.single(headers, "x-amz-copy-source"); if (source == null) throw new StoreException(400, "InvalidArgument", "Missing copy source"); if (source.startsWith("/")) source = source.substring(1); + String sourceVersion = null; + int question = source.indexOf('?'); + if (question >= 0) { + Map sourceQuery = query(source.substring(question + 1)); + if (sourceQuery.size() != 1 || !sourceQuery.containsKey("versionId")) + throw new StoreException(400, "InvalidArgument", "Invalid copy source version"); + sourceVersion = sourceQuery.get("versionId"); + source = source.substring(0, question); + } int separator = source.indexOf('/'); - if (separator <= 0 || separator == source.length() - 1 || source.indexOf('?') >= 0) + if (separator <= 0 || separator == source.length() - 1) throw new StoreException(400, "InvalidArgument", "Invalid copy source"); String sourceBucket = SigV4.decode(source.substring(0, separator)); String sourceKey = SigV4.decode(source.substring(separator + 1)); - if (!sourceBucket.equals(bucket)) throw new StoreException(404, "NoSuchBucket", "Bucket not found"); + store.bucket(sourceBucket); if (sourceKey.isEmpty() || sourceKey.getBytes(StandardCharsets.UTF_8).length > 1024 || sourceKey.indexOf('\0') >= 0) throw new StoreException(400, "InvalidArgument", "Invalid copy source key"); @@ -198,23 +499,146 @@ public final class Main { throw new StoreException(400, "InvalidArgument", "Invalid metadata directive"); if (!"REPLACE".equals(directive) && headers.containsKey("content-type")) unsupported("Content-Type requires REPLACE metadata directive"); - try (var object = store.open(bucket, sourceKey)) { + String tagDirective = SigV4.single(headers, "x-amz-tagging-directive"); + if (tagDirective != null && !tagDirective.equals("COPY") && !tagDirective.equals("REPLACE")) + throw new StoreException(400, "InvalidArgument", "Invalid tagging directive"); + if (!"REPLACE".equals(tagDirective) && headers.containsKey("x-amz-tagging")) + unsupported("Tagging header requires REPLACE tagging directive"); + if (!"REPLACE".equals(directive) && !ObjectAttributes.userMetadata(headers).isEmpty()) + unsupported("User metadata requires REPLACE metadata directive"); + try (var object = store.open(sourceBucket, sourceKey, sourceVersion)) { var sourceMetadata = object.metadata(); + Acl.require(sourceMetadata.acl(), principal, owner, Acl.READ); + if (sourceMetadata.versionId() != null) + exchange.getResponseHeaders().set("x-amz-copy-source-version-id", sourceMetadata.versionId()); String type = "REPLACE".equals(directive) ? contentType(headers) : sourceMetadata.contentType(); - var copied = store.put(bucket, key, object.stream(), sourceMetadata.length(), - SigV4.hex(sourceMetadata.sha256()), null, false, type); + Map metadata = "REPLACE".equals(directive) + ? ObjectAttributes.userMetadata(headers) : sourceMetadata.userMetadata(); + Map tags = "REPLACE".equals(tagDirective) + ? ObjectAttributes.tagsHeader(SigV4.single(headers, "x-amz-tagging")) : sourceMetadata.tags(); + Map acl = Acl.fromHeaders(headers, authentication.identities()); + if (!acl.isEmpty() && !owner.equals(principal)) + requireBucket(bucketName, principal, Acl.WRITE_ACP); + var copied = store.put(bucketName, key, object.stream(), sourceMetadata.length(), + SigV4.hex(sourceMetadata.sha256()), null, false, type, metadata, tags, + sourceMetadata::checksums, acl); + if (copied.versionId() != null) + exchange.getResponseHeaders().set("x-amz-version-id", copied.versionId()); sendXml(exchange, 200, "" + Instant.ofEpochMilli(copied.modified()) + """ + copied.etag() + """); } } - private void deleteObject(HttpExchange exchange, String key) throws IOException { + private void deleteObject(HttpExchange exchange, String key, String bucketName, + String versionId) throws IOException { if (exchange.getRequestHeaders().containsKey("if-none-match")) unsupported("Conditional delete"); - store.delete(bucket, key); + var result = store.delete(bucketName, key, versionId); + if (result.versionId() != null) + exchange.getResponseHeaders().set("x-amz-version-id", result.versionId()); + if (result.deleteMarker()) exchange.getResponseHeaders().set("x-amz-delete-marker", "true"); exchange.sendResponseHeaders(204, -1); } + private void handleTagging(HttpExchange exchange, String method, String key, + String hash, String bucketName, String versionId) throws IOException { + if (exchange.getRequestHeaders().containsKey("x-amz-tagging") || + exchange.getRequestHeaders().containsKey("x-amz-tagging-directive") || + exchange.getRequestHeaders().containsKey("x-amz-sdk-checksum-algorithm") || + !method.equals("PUT") && exchange.getRequestHeaders().containsKey("content-md5") || + exchange.getRequestHeaders().keySet().stream().anyMatch(name -> + name.toLowerCase(Locale.ROOT).startsWith("x-amz-checksum-")) || + !ObjectAttributes.userMetadata(exchange.getRequestHeaders()).isEmpty()) + unsupported("Tagging request header"); + switch (method) { + case "GET" -> { + requireEmptyBody(exchange, hash); + if (versionId != null) exchange.getResponseHeaders().set("x-amz-version-id", versionId); + sendXml(exchange, 200, tagXml(store.tags(bucketName, key, versionId))); + } + case "PUT" -> { + byte[] body = signedBody(exchange, hash, 65536); + verifyContentMd5(exchange.getRequestHeaders(), body); + store.setTags(bucketName, key, versionId, parseTags(body)); + if (versionId != null) exchange.getResponseHeaders().set("x-amz-version-id", versionId); + exchange.sendResponseHeaders(200, -1); + } + case "DELETE" -> { + requireEmptyBody(exchange, hash); + store.setTags(bucketName, key, versionId, Map.of()); + if (versionId != null) exchange.getResponseHeaders().set("x-amz-version-id", versionId); + exchange.sendResponseHeaders(204, -1); + } + default -> unsupported("Tagging operation"); + } + } + + private static void verifyContentMd5(com.sun.net.httpserver.Headers headers, byte[] body) { + String encoded = SigV4.single(headers, "content-md5"); + if (encoded == null) return; + byte[] expected; + try { expected = Base64.getDecoder().decode(encoded); } + catch (IllegalArgumentException error) { + throw new StoreException(400, "InvalidDigest", "Invalid Content-MD5"); + } + if (expected.length != 16) throw new StoreException(400, "InvalidDigest", "Invalid Content-MD5"); + try { + byte[] actual = MessageDigest.getInstance("MD5").digest(body); + if (!MessageDigest.isEqual(expected, actual)) + throw new StoreException(400, "BadDigest", "Content-MD5 mismatch"); + } catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); } + } + + private static String tagXml(Map tags) { + StringBuilder body = new StringBuilder(""); + new java.util.TreeMap<>(tags).forEach((key, value) -> body.append("") + .append(xml(key)).append("").append(xml(value)).append("")); + return body.append("").toString(); + } + + private static Map parseTags(byte[] body) { + try { + var factory = javax.xml.parsers.DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setExpandEntityReferences(false); + var builder = factory.newDocumentBuilder(); + builder.setErrorHandler(new org.xml.sax.helpers.DefaultHandler() { + @Override public void fatalError(org.xml.sax.SAXParseException error) throws org.xml.sax.SAXException { + throw error; + } + }); + var document = builder.parse(new ByteArrayInputStream(body)); + var root = document.getDocumentElement(); + if (!root.getTagName().equals("Tagging")) throw new IllegalArgumentException(); + var sets = root.getElementsByTagName("TagSet"); + if (sets.getLength() != 1) throw new IllegalArgumentException(); + for (int i = 0; i < root.getChildNodes().getLength(); i++) { + var child = root.getChildNodes().item(i); + if (child instanceof org.w3c.dom.Element && child != sets.item(0)) + throw new IllegalArgumentException(); + } + Map tags = new java.util.TreeMap<>(); + var entries = sets.item(0).getChildNodes(); + for (int i = 0; i < entries.getLength(); i++) { + if (!(entries.item(i) instanceof org.w3c.dom.Element tag)) continue; + if (!tag.getTagName().equals("Tag")) throw new IllegalArgumentException(); + var keys = tag.getElementsByTagName("Key"); + var values = tag.getElementsByTagName("Value"); + if (keys.getLength() != 1 || values.getLength() != 1 || + tag.getElementsByTagName("*").getLength() != 2) throw new IllegalArgumentException(); + if (tags.put(keys.item(0).getTextContent(), values.item(0).getTextContent()) != null) + throw new IllegalArgumentException(); + } + ObjectAttributes.validateTags(tags); + return Map.copyOf(tags); + } catch (Exception error) { + throw new StoreException(400, "MalformedXML", "Invalid object tagging body"); + } + } + private static String contentType(com.sun.net.httpserver.Headers headers) { String value = SigV4.single(headers, "content-type"); if (value == null) return "application/octet-stream"; @@ -228,7 +652,7 @@ public final class Main { var headers = exchange.getRequestHeaders(); if (headers.containsKey("transfer-encoding") || (headers.containsKey("content-length") && !"0".equals(SigV4.single(headers, "content-length"))) || - !hash.equals(SigV4.hex(SigV4.hash(new byte[0])))) + !(hash.equals("UNSIGNED-PAYLOAD") || hash.equals(SigV4.hex(SigV4.hash(new byte[0]))))) throw new StoreException(400, "InvalidRequest", "Request must have an empty body"); } @@ -253,20 +677,33 @@ public final class Main { } private void handleMultipart(HttpExchange exchange, String method, Map query, - String key, String hash) throws IOException { + String key, SigV4.Verified verified, String bucketName) throws IOException { + String hash = verified.payload(); var headers = exchange.getRequestHeaders(); - if (headers.containsKey("content-encoding") || headers.containsKey("if-none-match")) + if ((headers.containsKey("content-encoding") && !(method.equals("PUT") && verified.streaming())) || + headers.containsKey("if-none-match")) unsupported("Multipart request header"); - if (headers.containsKey("x-amz-copy-source") || headers.containsKey("x-amz-metadata-directive")) + if (headers.containsKey("x-amz-copy-source") || headers.containsKey("x-amz-metadata-directive") || + headers.containsKey("x-amz-tagging-directive")) unsupported("Multipart copy request"); + if (!query.containsKey("uploads") && + (headers.containsKey("x-amz-tagging") || !ObjectAttributes.userMetadata(headers).isEmpty() || + headers.containsKey("x-amz-acl") || headers.keySet().stream().anyMatch( + name -> name.toLowerCase(Locale.ROOT).startsWith("x-amz-grant-")))) + unsupported("Multipart object attributes belong on initiation"); if (!method.equals("PUT") && (headers.containsKey("content-md5") || headers.containsKey("x-amz-sdk-checksum-algorithm") || headers.keySet().stream().anyMatch(name -> name.toLowerCase(Locale.ROOT).startsWith("x-amz-checksum-")))) unsupported("Multipart checksum header"); if (query.containsKey("uploads")) { requireEmptyBody(exchange, hash); - String id = multipart.create(bucket, key, contentType(headers)); - sendXml(exchange, 200, "" + xml(bucket) + + Map acl = Acl.fromHeaders(headers, authentication.identities()); + if (!acl.isEmpty() && !owner.equals(verified.principal())) + requireBucket(bucketName, verified.principal(), Acl.WRITE_ACP); + String id = multipart.create(bucketName, key, contentType(headers), + ObjectAttributes.userMetadata(headers), + ObjectAttributes.tagsHeader(SigV4.single(headers, "x-amz-tagging")), acl); + sendXml(exchange, 200, "" + xml(bucketName) + "" + xml(key) + "" + id + ""); return; @@ -277,40 +714,47 @@ public final class Main { int number; try { number = Integer.parseInt(query.get("partNumber")); } catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid part number"); } - long length = contentLength(headers); + long length = verified.streaming() ? decodedLength(headers) : contentLength(headers); UploadChecksums checksums = UploadChecksums.from(headers); - String etag = multipart.putPart(id, bucket, key, number, - checksums.verifying(exchange.getRequestBody()), length, hash, checksums.sha256()); + AwsChunkedInputStream chunked = verified.streaming() ? chunkedBody(exchange, verified) : null; + String etag = multipart.putPart(id, bucketName, key, number, + checksums.verifying(chunked == null ? exchange.getRequestBody() : chunked), + length, expectedPayloadHash(hash), checksums.sha256()); exchange.getResponseHeaders().set("ETag", "\"" + etag + "\""); checksums.response(exchange.getResponseHeaders()); + if (chunked != null && chunked.trailerValue() != null) + exchange.getResponseHeaders().set(SigV4.single(headers, "x-amz-trailer"), chunked.trailerValue()); exchange.sendResponseHeaders(200, -1); } case "POST" -> { byte[] body = signedBody(exchange, hash, 65536); List parts = completedParts(body); - var meta = multipart.complete(id, bucket, key, parts); - sendXml(exchange, 200, "" + xml(bucket) + + var meta = multipart.complete(id, bucketName, key, parts); + if (meta.versionId() != null) + exchange.getResponseHeaders().set("x-amz-version-id", meta.versionId()); + sendXml(exchange, 200, "" + xml(bucketName) + "" + xml(key) + """ + meta.etag() + """); } case "DELETE" -> { requireEmptyBody(exchange, hash); - multipart.abort(id, bucket, key); + multipart.abort(id, bucketName, key); exchange.sendResponseHeaders(204, -1); } case "GET" -> { requireEmptyBody(exchange, hash); - listParts(exchange, id, key, query); + listParts(exchange, id, key, query, bucketName); } default -> unsupported("Multipart operation"); } } - private void listParts(HttpExchange exchange, String id, String key, Map query) throws IOException { + private void listParts(HttpExchange exchange, String id, String key, Map query, + String bucketName) throws IOException { int marker = boundedNumber(query.get("part-number-marker"), 0, 10000, 0); int maxParts = boundedNumber(query.get("max-parts"), 1, 1000, 1000); - MultipartStorage.PartPage page = multipart.listParts(id, bucket, key, marker, maxParts); - StringBuilder body = new StringBuilder("").append(xml(bucket)) + MultipartStorage.PartPage page = multipart.listParts(id, bucketName, key, marker, maxParts); + StringBuilder body = new StringBuilder("").append(xml(bucketName)) .append("").append(xml(key)).append("").append(xml(id)) .append("").append(marker) .append("").append(page.nextMarker()) @@ -324,14 +768,14 @@ public final class Main { sendXml(exchange, 200, body.append("").toString()); } - private void listUploads(HttpExchange exchange, Map query) throws IOException { + private void listUploads(HttpExchange exchange, Map query, String bucketName) throws IOException { String prefix = query.getOrDefault("prefix", ""); String marker = query.getOrDefault("key-marker", ""); String uploadMarker = query.getOrDefault("upload-id-marker", ""); if (!uploadMarker.isEmpty() && marker.isEmpty()) throw new StoreException(400, "InvalidArgument", "Upload ID marker requires a key marker"); int maximum = boundedNumber(query.get("max-uploads"), 1, 1000, 1000); - List uploads = multipart.listUploads(bucket, prefix); + List uploads = multipart.listUploads(bucketName, prefix); List page = new ArrayList<>(); boolean truncated = false; for (MultipartStorage.UploadInfo upload : uploads) { @@ -343,7 +787,7 @@ public final class Main { } page.add(upload); } - StringBuilder body = new StringBuilder("").append(xml(bucket)) + StringBuilder body = new StringBuilder("").append(xml(bucketName)) .append("").append(xml(marker)).append("") .append(xml(uploadMarker)).append("").append(maximum) .append("").append(truncated).append(""); @@ -373,19 +817,52 @@ public final class Main { return value; } + private static long decodedLength(com.sun.net.httpserver.Headers headers) { + String value = SigV4.single(headers, "x-amz-decoded-content-length"); + if (value == null || !value.matches("[0-9]{1,19}")) + throw new StoreException(400, "InvalidArgument", "Invalid decoded content length"); + try { return Long.parseLong(value); } + catch (NumberFormatException error) { + throw new StoreException(400, "InvalidArgument", "Invalid decoded content length"); + } + } + + private static AwsChunkedInputStream chunkedBody(HttpExchange exchange, + SigV4.Verified verified) { + var headers = exchange.getRequestHeaders(); + if (!"aws-chunked".equals(SigV4.single(headers, "content-encoding"))) + throw new StoreException(400, "InvalidRequest", "Signed chunk upload requires aws-chunked encoding"); + if (contentLength(headers) < 0 && !headers.containsKey("transfer-encoding")) + throw new StoreException(411, "MissingContentLength", "Encoded content length is required"); + long length = decodedLength(headers); + String trailer = SigV4.single(headers, "x-amz-trailer"); + if (verified.payload().endsWith("-TRAILER")) { + if (trailer == null || !trailer.matches("x-amz-checksum-[a-z0-9]+")) + throw new StoreException(400, "InvalidRequest", "A checksum trailer is required"); + } else if (trailer != null) { + throw new StoreException(400, "InvalidRequest", "Unexpected checksum trailer"); + } + return new AwsChunkedInputStream(exchange.getRequestBody(), verified, length, trailer); + } + private static long contentLength(com.sun.net.httpserver.Headers headers) { String text = SigV4.single(headers, "content-length"); if (text == null) return -1; try { return Long.parseLong(text); } catch (NumberFormatException e) { throw new StoreException(400, "InvalidArgument", "Invalid Content-Length"); } } + private static String expectedPayloadHash(String hash) { + return hash.matches("[0-9a-f]{64}") ? hash : null; + } private static byte[] signedBody(HttpExchange exchange, String hash, int limit) throws IOException { + if (!hash.equals("UNSIGNED-PAYLOAD") && !hash.matches("[0-9a-f]{64}")) + throw new StoreException(400, "InvalidRequest", "Unsupported body signing mode"); long length = contentLength(exchange.getRequestHeaders()); if (length < 0) throw new StoreException(411, "MissingContentLength", "Content-Length is required"); if (length > limit) throw new StoreException(413, "EntityTooLarge", "Request body is too large"); byte[] body = exchange.getRequestBody().readNBytes(limit + 1); if (body.length != length) throw new StoreException(400, "IncompleteBody", "Body length does not match Content-Length"); - if (!SigV4.hex(SigV4.hash(body)).equals(hash)) + if (!hash.equals("UNSIGNED-PAYLOAD") && !SigV4.hex(SigV4.hash(body)).equals(hash)) throw new StoreException(400, "XAmzContentSHA256Mismatch", "Payload hash mismatch"); return body; } @@ -430,12 +907,17 @@ public final class Main { exchange.getResponseBody().write(body); } - private void readObject(HttpExchange exchange, String key) throws IOException { + private void readObject(HttpExchange exchange, String key, String bucketName, + String versionId, String principal) throws IOException { var headers = exchange.getRequestHeaders(); + String checksumMode = SigV4.single(headers, "x-amz-checksum-mode"); + if (checksumMode != null && !checksumMode.equals("ENABLED")) + throw new StoreException(400, "InvalidArgument", "Invalid checksum mode"); if (headers.containsKey("if-match") || headers.containsKey("if-modified-since") || headers.containsKey("if-unmodified-since")) unsupported("Conditional read"); - try (var object = store.open(bucket, key)) { + try (var object = store.open(bucketName, key, versionId)) { var meta = object.metadata(); + Acl.require(meta.acl(), principal, owner, Acl.READ); String etag = "\"" + meta.etag() + "\""; String noneMatch = SigV4.single(headers, "if-none-match"); if (noneMatch != null && (noneMatch.equals("*") || @@ -455,8 +937,13 @@ public final class Main { response.set("Content-Length", Long.toString(range.length())); response.set("Accept-Ranges", "bytes"); response.set("ETag", etag); + if (meta.versionId() != null) response.set("x-amz-version-id", meta.versionId()); response.set("Last-Modified", DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) .format(Instant.ofEpochMilli(meta.modified()))); + meta.userMetadata().forEach((name, value) -> response.set("x-amz-meta-" + name, value)); + if (checksumMode != null) + meta.checksums().forEach(response::set); + if (!meta.tags().isEmpty()) response.set("x-amz-tagging-count", Integer.toString(meta.tags().size())); if (range.partial()) response.set("Content-Range", "bytes " + range.start() + "-" + range.end() + "/" + meta.length()); int status = range.partial() ? 206 : 200; if (exchange.getRequestMethod().equals("HEAD") || range.length() == 0) @@ -502,11 +989,95 @@ public final class Main { } } - private void listObjects(HttpExchange exchange, Map query) throws IOException { + private void handleVersioning(HttpExchange exchange, String hash, String bucketName) throws IOException { + switch (exchange.getRequestMethod()) { + case "GET" -> { + requireEmptyBody(exchange, hash); + var state = store.bucket(bucketName).versioning(); + String status = state == ObjectStorage.VersioningState.NEVER ? "" : + "" + state.name().charAt(0) + state.name().substring(1).toLowerCase(Locale.ROOT) + + ""; + sendXml(exchange, 200, "" + + status + ""); + } + case "PUT" -> { + byte[] body = signedBody(exchange, hash, 65536); + verifyContentMd5(exchange.getRequestHeaders(), body); + store.setVersioning(bucketName, parseVersioning(body)); + exchange.sendResponseHeaders(200, -1); + } + default -> unsupported("Bucket versioning operation"); + } + } + + private static ObjectStorage.VersioningState parseVersioning(byte[] body) { + try { + var factory = javax.xml.parsers.DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setExpandEntityReferences(false); + var document = factory.newDocumentBuilder().parse(new ByteArrayInputStream(body)); + var root = document.getDocumentElement(); + if (!root.getNodeName().equals("VersioningConfiguration")) throw new IllegalArgumentException(); + String status = null; + for (int i = 0; i < root.getChildNodes().getLength(); i++) { + var child = root.getChildNodes().item(i); + if (!(child instanceof org.w3c.dom.Element element)) continue; + if (!element.getNodeName().equals("Status") || status != null) + throw new IllegalArgumentException(); + status = element.getTextContent().trim(); + } + if ("Enabled".equals(status)) return ObjectStorage.VersioningState.ENABLED; + if ("Suspended".equals(status)) return ObjectStorage.VersioningState.SUSPENDED; + } catch (Exception ignored) { } + throw new StoreException(400, "MalformedXML", "Invalid versioning configuration"); + } + + private void listVersions(HttpExchange exchange, Map query, + String bucketName) throws IOException { + String encoding = query.get("encoding-type"); + if (encoding != null && !encoding.equals("url")) unsupported("Encoding type"); + int maximum = boundedNumber(query.get("max-keys"), 0, 1000, 1000); + String keyMarker = query.get("key-marker"); + String versionMarker = query.get("version-id-marker"); + if (versionMarker != null && keyMarker == null) + throw new StoreException(400, "InvalidArgument", "Version ID marker requires a key marker"); + var page = store.listVersions(bucketName, query.getOrDefault("prefix", ""), + keyMarker, versionMarker, maximum); + StringBuilder body = new StringBuilder("") + .append(xml(bucketName)).append("") + .append(xml(listKey(query.getOrDefault("prefix", ""), encoding))).append("") + .append(xml(listKey(query.getOrDefault("key-marker", ""), encoding))) + .append("") + .append(xml(query.getOrDefault("version-id-marker", ""))) + .append("").append(maximum) + .append("").append(page.truncated()).append(""); + if (encoding != null) body.append("url"); + if (page.truncated()) body.append("") + .append(xml(listKey(page.nextKey(), encoding))).append("") + .append(xml(page.nextVersionId())).append(""); + for (var entry : page.entries()) { + body.append(entry.deleteMarker() ? "" : "") + .append("").append(xml(listKey(entry.key(), encoding))) + .append("").append(xml(entry.versionId())) + .append("").append(entry.latest()) + .append("") + .append(Instant.ofEpochMilli(entry.modified())).append(""); + if (!entry.deleteMarker()) body.append(""").append(entry.metadata().etag()) + .append(""").append(entry.metadata().length()).append(""); + body.append("objectstoreObjectStore") + .append(entry.deleteMarker() ? "" : "STANDARD"); + } + sendXml(exchange, 200, body.append("").toString()); + } + + private void listObjects(HttpExchange exchange, Map query, String bucketName) throws IOException { ListRequest request = listRequest(query); - var page = store.list(bucket, request.prefix(), request.delimiter(), request.maxKeys(), request.after()); + var page = store.list(bucketName, request.prefix(), request.delimiter(), request.maxKeys(), request.after()); StringBuilder xml = new StringBuilder(""); - appendListHeader(xml, query, request, page); + appendListHeader(xml, query, request, page, bucketName); appendListEntries(xml, page, request.encoding()); if (page.truncated()) xml.append("") .append(Base64.getUrlEncoder().withoutPadding().encodeToString(page.nextKey().getBytes(StandardCharsets.UTF_8))) @@ -543,9 +1114,10 @@ public final class Main { return new ListRequest(prefix, delimiter, encoding, maxKeys, after); } - private void appendListHeader(StringBuilder xml, Map query, ListRequest request, ObjectStorage.ListPage page) { + private void appendListHeader(StringBuilder xml, Map query, ListRequest request, + ObjectStorage.ListPage page, String bucketName) { String prefix = request.prefix(), delimiter = request.delimiter(), encoding = request.encoding(); - xml.append("").append(xml(bucket)).append("").append(xml(listKey(prefix, encoding))).append(""); + xml.append("").append(xml(bucketName)).append("").append(xml(listKey(prefix, encoding))).append(""); if (!delimiter.isEmpty()) xml.append("").append(xml(listKey(delimiter, encoding))).append(""); if (encoding != null) xml.append("url"); if (query.containsKey("continuation-token")) xml.append("") @@ -633,7 +1205,8 @@ public final class Main { store = disk; multipart = new MultipartStore(disk); } else throw new IllegalArgumentException("Invalid STORE_MODE"); - var app = new Main(store, multipart, new SigV4(access, secret, region, Clock.systemUTC()), bucket); + var app = new Main(store, multipart, new SigV4(identities(env, access, secret), + access, region, Clock.systemUTC()), bucket, ClientLimits.fromEnvironment(env)); int port = Integer.parseInt(env.getOrDefault("PORT", "9000")); var server = HttpServer.create(mode.equals("cluster") ? new InetSocketAddress(env.getOrDefault("BIND_ADDRESS", "127.0.0.1"), port) @@ -651,6 +1224,27 @@ public final class Main { if (store instanceof DiskStore disk) printStartup(disk, app.multipart, bucket, region, port); else System.out.println("ObjectStore local cluster prototype v" + Version.VALUE + " listening on :" + port); } + + static Map identities(Map env, String access, String secret) + throws IOException { + Map values = new HashMap<>(); + values.put(access, secret); + String file = env.get("S3_CREDENTIALS_FILE"); + if (file == null || file.isBlank()) return values; + List lines = java.nio.file.Files.readAllLines(Path.of(file), StandardCharsets.UTF_8); + if (lines.size() > 63) throw new IllegalArgumentException("Too many additional identities"); + for (String line : lines) { + if (line.isBlank() || line.startsWith("#")) continue; + int separator = line.indexOf(':'); + if (separator < 0) throw new IllegalArgumentException("Invalid additional credential entry"); + String key = line.substring(0, separator); + String value = line.substring(separator + 1); + if (!key.matches("[A-Za-z0-9]{16,128}") || value.length() < 32 || + values.putIfAbsent(key, value) != null) + throw new IllegalArgumentException("Invalid or duplicate additional credential"); + } + return values; + } private static void printStartup(DiskStore store, MultipartStorage multipart, String bucket, String region, int port) { System.out.println(" *"); diff --git a/src/cloud/lunarsky/store/MultipartStorage.java b/src/cloud/lunarsky/store/MultipartStorage.java index fdf0d59..ce660b5 100644 --- a/src/cloud/lunarsky/store/MultipartStorage.java +++ b/src/cloud/lunarsky/store/MultipartStorage.java @@ -3,6 +3,7 @@ package cloud.lunarsky.store; import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; interface MultipartStorage { record Part(int number, String etag) {} @@ -10,7 +11,16 @@ interface MultipartStorage { record PartPage(List parts, int nextMarker, boolean truncated) {} record UploadInfo(String id, String key, long created) {} - String create(String bucket, String key, String contentType) throws IOException; + String create(String bucket, String key, String contentType, + Map userMetadata, Map tags, + Map acl) throws IOException; + default String create(String bucket, String key, String contentType, + Map userMetadata, Map tags) throws IOException { + return create(bucket, key, contentType, userMetadata, tags, Map.of()); + } + default String create(String bucket, String key, String contentType) throws IOException { + return create(bucket, key, contentType, Map.of(), Map.of()); + } String putPart(String id, String bucket, String key, int number, InputStream input, long length, String expectedHash, String checksum) throws IOException; ObjectStorage.Metadata complete(String id, String bucket, String key, List parts) throws IOException; diff --git a/src/cloud/lunarsky/store/MultipartStore.java b/src/cloud/lunarsky/store/MultipartStore.java index e9684af..6c236e1 100644 --- a/src/cloud/lunarsky/store/MultipartStore.java +++ b/src/cloud/lunarsky/store/MultipartStore.java @@ -8,12 +8,16 @@ import cloud.lunarsky.store.MultipartStorage.Part; final class MultipartStore implements MultipartStorage { private static final int MAGIC = 0x4c534d50; + private static final int MAGIC_V2 = 0x4c534d51; + private static final int MAGIC_V3 = 0x4c534d52; private final DiskStore store; private final Path root; private long staged; private int active; - private record Upload(String bucket, String key, String contentType) {} + private record Upload(String bucket, String key, String contentType, + Map userMetadata, Map tags, + Map acl) {} MultipartStore(DiskStore store) throws IOException { this.store = store; @@ -42,17 +46,29 @@ final class MultipartStore implements MultipartStorage { if (staged > store.maxTotal()) throw new IOException("Multipart staging limit exceeded"); } - public synchronized String create(String bucket, String key, String contentType) throws IOException { + public synchronized String create(String bucket, String key, String contentType, + Map userMetadata, Map tags, + Map acl) throws IOException { + if (Files.exists(store.root().resolve("buckets.bin"))) store.bucket(bucket); if (active >= 32) throw new StoreException(503, "SlowDown", "Too many active uploads"); String id = UUID.randomUUID().toString(); Path pending = root.resolve(".creating-" + id), dir = root.resolve(id); Files.createDirectory(pending); try { try (var output = new DataOutputStream(Files.newOutputStream(pending.resolve("manifest"), StandardOpenOption.CREATE_NEW))) { - output.writeInt(MAGIC); + output.writeInt(MAGIC_V3); output.writeUTF(bucket); output.writeUTF(key); output.writeUTF(contentType); + byte[] custom = ObjectAttributes.encode(userMetadata, 4096); + byte[] encodedTags = ObjectAttributes.encode(tags, 8192); + output.writeShort(custom.length); + output.write(custom); + output.writeShort(encodedTags.length); + output.write(encodedTags); + byte[] encodedAcl = ObjectAttributes.encode(acl, 2048); + output.writeShort(encodedAcl.length); + output.write(encodedAcl); } try (var channel = java.nio.channels.FileChannel.open(pending.resolve("manifest"), StandardOpenOption.READ)) { channel.force(true); @@ -109,7 +125,7 @@ final class MultipartStore implements MultipartStorage { } if (count != length) throw new StoreException(400, "IncompleteBody", "Part length does not match Content-Length"); byte[] actual = sha.digest(); - if (!MessageDigest.isEqual(actual, HexFormat.of().parseHex(expectedHash))) + if (expectedHash != null && !MessageDigest.isEqual(actual, HexFormat.of().parseHex(expectedHash))) throw new StoreException(400, "XAmzContentSHA256Mismatch", "Part hash mismatch"); if (checksum != null && !Base64.getEncoder().encodeToString(actual).equals(checksum)) throw new StoreException(400, "BadDigest", "SHA-256 checksum mismatch"); @@ -153,8 +169,9 @@ final class MultipartStore implements MultipartStorage { } ObjectStorage.Metadata result; try (InputStream input = new PartsInput(paths)) { + Upload upload = readUpload(dir); result = store.put(bucket, key, input, total, SigV4.hex(sha.digest()), null, false, - readUpload(dir).contentType()); + upload.contentType(), upload.userMetadata(), upload.tags(), Map::of, upload.acl()); } remove(dir); return result; @@ -217,8 +234,14 @@ final class MultipartStore implements MultipartStorage { } private static Upload readUpload(Path dir) throws IOException { try (var input = new DataInputStream(Files.newInputStream(dir.resolve("manifest")))) { - if (input.readInt() != MAGIC) throw new IOException("Invalid multipart upload manifest"); - Upload upload = new Upload(input.readUTF(), input.readUTF(), input.readUTF()); + int magic = input.readInt(); + if (magic != MAGIC && magic != MAGIC_V2 && magic != MAGIC_V3) + throw new IOException("Invalid multipart upload manifest"); + String bucket = input.readUTF(), key = input.readUTF(), type = input.readUTF(); + Map custom = magic != MAGIC ? ObjectAttributes.decode(input.readNBytes(input.readUnsignedShort())) : Map.of(); + Map tags = magic != MAGIC ? ObjectAttributes.decode(input.readNBytes(input.readUnsignedShort())) : Map.of(); + Map acl = magic == MAGIC_V3 ? ObjectAttributes.decode(input.readNBytes(input.readUnsignedShort())) : Map.of(); + Upload upload = new Upload(bucket, key, type, custom, tags, acl); if (input.read() != -1) throw new IOException("Invalid multipart upload manifest"); return upload; } diff --git a/src/cloud/lunarsky/store/NodeClient.java b/src/cloud/lunarsky/store/NodeClient.java index a0ace8c..a8a9722 100644 --- a/src/cloud/lunarsky/store/NodeClient.java +++ b/src/cloud/lunarsky/store/NodeClient.java @@ -14,6 +14,8 @@ import java.util.HexFormat; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; final class NodeClient { record Node(UUID id, UUID hostId, URI url) {} @@ -25,6 +27,10 @@ final class NodeClient { private final String token; private final String repairToken; private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(3)).build(); + private final ConcurrentHashMap unreadableUntil = new ConcurrentHashMap<>(); + private final ConcurrentHashMap healthyUntil = new ConcurrentHashMap<>(); + private static final long READ_RETRY_NANOS = TimeUnit.SECONDS.toNanos(5); + private static final long HEALTH_FRESH_NANOS = TimeUnit.SECONDS.toNanos(3); NodeClient(List nodes, String token, String repairToken) { if (nodes.isEmpty() || nodes.stream().map(Node::id).distinct().count() != nodes.size() || @@ -97,14 +103,33 @@ final class NodeClient { for (int i = 0; i < nodes.size(); i++) { Node node = nodes.get(i); NodeIdentity actual = probeIfAvailable(node.url(), token); - if (actual == null) continue; - if (actual.nodeId().equals(node.id()) && actual.hostId().equals(node.hostId())) - healthy.add(faultDomain(i, testNodeDomains)); + if (actual == null || !actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())) { + markUnreadable(node); + continue; + } + unreadableUntil.remove(node.id()); + healthyUntil.put(node.id(), System.nanoTime() + HEALTH_FRESH_NANOS); + healthy.add(faultDomain(i, testNodeDomains)); if (healthy.size() >= required) return true; } return false; } + private void markUnreadable(Node node) { + healthyUntil.remove(node.id()); + unreadableUntil.put(node.id(), System.nanoTime() + READ_RETRY_NANOS); + } + + private boolean unreadable(Node node) { + Long until = unreadableUntil.get(node.id()); + return until != null && System.nanoTime() - until < 0; + } + + private boolean recentlyHealthy(Node node) { + Long until = healthyUntil.get(node.id()); + return until != null && System.nanoTime() - until < 0; + } + void put(int index, UUID id, byte[] data, byte[] sha256) throws IOException { put(index, id, data, sha256, false); } @@ -132,18 +157,39 @@ final class NodeClient { byte[] get(int index, UUID id, int length, byte[] sha256) throws IOException { if (length < 1 || length > ClusterNode.MAX_SEGMENT) throw new IOException("Invalid segment length"); Node node = nodes.get(index); + if (unreadable(node)) throw new IOException("Storage node is temporarily unreachable"); + if (!recentlyHealthy(node)) { + NodeIdentity actual = probeIfAvailable(node.url(), token); + if (actual == null || !actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())) { + markUnreadable(node); + throw new IOException("Storage node is temporarily unreachable"); + } + healthyUntil.put(node.id(), System.nanoTime() + HEALTH_FRESH_NANOS); + } HttpRequest request = HttpRequest.newBuilder(node.url().resolve("/segments/" + id)) .timeout(Duration.ofSeconds(30)).header("X-Cluster-Token", token) .header("X-Cluster-Expected-Node", node.id().toString()).GET().build(); - HttpResponse response = send(request, HttpResponse.BodyHandlers.ofInputStream()); - try (InputStream body = response.body()) { - if (response.statusCode() != 200) - throw new IOException("Node " + node.id() + " has no verified copy of segment " + id); - byte[] bytes = body.readNBytes(length + 1); - if (bytes.length != length || !MessageDigest.isEqual(SigV4.hash(bytes), sha256)) - throw new IOException("Node " + node.id() + " has no verified copy of segment " + id); - return bytes; + HttpResponse response; + try { response = send(request, HttpResponse.BodyHandlers.ofInputStream()); } + catch (IOException error) { + markUnreadable(node); + throw error; } + if (response.statusCode() != 200) { + response.body().close(); + throw new IOException("Node " + node.id() + " has no verified copy of segment " + id); + } + byte[] bytes; + try (InputStream body = response.body()) { bytes = body.readNBytes(length + 1); } + catch (IOException error) { + markUnreadable(node); + throw error; + } + if (bytes.length != length || !MessageDigest.isEqual(SigV4.hash(bytes), sha256)) + throw new IOException("Node " + node.id() + " has no verified copy of segment " + id); + unreadableUntil.remove(node.id()); + healthyUntil.put(node.id(), System.nanoTime() + HEALTH_FRESH_NANOS); + return bytes; } List inventory(int index, String shard, UUID after) throws IOException { diff --git a/src/cloud/lunarsky/store/ObjectAttributes.java b/src/cloud/lunarsky/store/ObjectAttributes.java new file mode 100644 index 0000000..84f4ac5 --- /dev/null +++ b/src/cloud/lunarsky/store/ObjectAttributes.java @@ -0,0 +1,114 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.Headers; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.TreeMap; + +final class ObjectAttributes { + private ObjectAttributes() {} + + static Map userMetadata(Headers headers) { + Map values = new TreeMap<>(); + int total = 0; + for (String name : headers.keySet()) { + if (!name.toLowerCase(java.util.Locale.ROOT).startsWith("x-amz-meta-")) continue; + String key = name.substring(11).toLowerCase(java.util.Locale.ROOT); + if (!key.matches("[a-z0-9][a-z0-9._-]{0,127}")) + throw new StoreException(400, "InvalidArgument", "Invalid user metadata key"); + String value = SigV4.single(headers, name); + if (value == null || !value.chars().allMatch(c -> c >= 32 && c <= 126)) + throw new StoreException(400, "InvalidArgument", "Invalid user metadata value"); + total += key.getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length; + values.put(key, value); + } + if (total > 2048) throw new StoreException(400, "MetadataTooLarge", "User metadata exceeds 2 KiB"); + return Map.copyOf(values); + } + + static Map tagsHeader(String raw) { + if (raw == null) return Map.of(); + Map tags = new TreeMap<>(); + if (raw.isEmpty()) return tags; + for (String pair : raw.split("&", -1)) { + String[] parts = pair.split("=", 2); + if (parts.length != 2) throw new StoreException(400, "InvalidTag", "Invalid tagging header"); + String key = SigV4.decode(parts[0]); + String value = SigV4.decode(parts[1]); + if (tags.put(key, value) != null) throw new StoreException(400, "InvalidTag", "Duplicate tag key"); + } + validateTags(tags); + return Map.copyOf(tags); + } + + static void validateTags(Map tags) { + if (tags.size() > 10) throw new StoreException(400, "InvalidTag", "Too many object tags"); + for (var entry : tags.entrySet()) { + if (entry.getKey().isEmpty() || entry.getKey().length() > 128 || + entry.getValue().length() > 256 || + !xmlText(entry.getKey()) || !xmlText(entry.getValue())) + throw new StoreException(400, "InvalidTag", "Invalid tag key or value"); + } + } + + private static boolean xmlText(String value) { + for (int i = 0; i < value.length();) { + int point = value.codePointAt(i); + if (point < 32 || point > 0x10ffff || point >= 0xd800 && point <= 0xdfff || + point >= 0xfffe && point <= 0xffff) return false; + i += Character.charCount(point); + } + return true; + } + + static byte[] encode(Map values, int limit) { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(bytes); + output.writeShort(values.size()); + for (var entry : new TreeMap<>(values).entrySet()) { + byte[] key = entry.getKey().getBytes(StandardCharsets.UTF_8); + byte[] value = entry.getValue().getBytes(StandardCharsets.UTF_8); + output.writeShort(key.length); + output.writeShort(value.length); + output.write(key); + output.write(value); + } + if (bytes.size() > limit) throw new StoreException(400, "InvalidArgument", "Object attributes are too large"); + return bytes.toByteArray(); + } catch (IOException error) { throw new IllegalStateException(error); } + } + + static Map decode(byte[] bytes) throws IOException { + if (bytes == null || bytes.length == 0) return Map.of(); + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes))) { + int count = input.readUnsignedShort(); + if (count > 128) throw new IOException("Invalid object attributes"); + Map values = new TreeMap<>(); + for (int i = 0; i < count; i++) { + int keyLength = input.readUnsignedShort(), valueLength = input.readUnsignedShort(); + String key = decodeUtf8(input.readNBytes(keyLength), keyLength); + String value = decodeUtf8(input.readNBytes(valueLength), valueLength); + if (values.put(key, value) != null) throw new IOException("Duplicate object attribute"); + } + if (input.available() != 0) throw new IOException("Trailing object attributes"); + return Map.copyOf(values); + } + } + + private static String decodeUtf8(byte[] bytes, int length) throws IOException { + if (bytes.length != length) throw new IOException("Truncated object attributes"); + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(java.nio.charset.CodingErrorAction.REPORT) + .decode(java.nio.ByteBuffer.wrap(bytes)).toString(); + } catch (java.nio.charset.CharacterCodingException error) { + throw new IOException("Invalid object attributes", error); + } + } +} diff --git a/src/cloud/lunarsky/store/ObjectStorage.java b/src/cloud/lunarsky/store/ObjectStorage.java index f0bfa81..00e194b 100644 --- a/src/cloud/lunarsky/store/ObjectStorage.java +++ b/src/cloud/lunarsky/store/ObjectStorage.java @@ -3,11 +3,46 @@ package cloud.lunarsky.store; import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; /** Storage operations shared by the local and cluster gateways. */ interface ObjectStorage extends AutoCloseable { + enum VersioningState { NEVER, ENABLED, SUSPENDED } + record Limits(long maxObjectBytes, long maxTotalBytes) {} + record Bucket(String name, long created, VersioningState versioning, Map acl) { + Bucket(String name, long created) { this(name, created, VersioningState.NEVER, Map.of()); } + Bucket(String name, long created, VersioningState versioning) { + this(name, created, versioning, Map.of()); + } + } record Metadata(long length, long modified, String etag, byte[] sha256, - String bucket, String key, String contentType) {} + String bucket, String key, String contentType, + Map userMetadata, Map tags, + String versionId, Map checksums, Map acl) { + Metadata(long length, long modified, String etag, byte[] sha256, + String bucket, String key, String contentType, + Map userMetadata, Map tags, + String versionId, Map checksums) { + this(length, modified, etag, sha256, bucket, key, contentType, + userMetadata, tags, versionId, checksums, Map.of()); + } + Metadata(long length, long modified, String etag, byte[] sha256, + String bucket, String key, String contentType, + Map userMetadata, Map tags, + String versionId) { + this(length, modified, etag, sha256, bucket, key, contentType, + userMetadata, tags, versionId, Map.of()); + } + Metadata(long length, long modified, String etag, byte[] sha256, + String bucket, String key, String contentType, + Map userMetadata, Map tags) { + this(length, modified, etag, sha256, bucket, key, contentType, userMetadata, tags, null); + } + Metadata(long length, long modified, String etag, byte[] sha256, + String bucket, String key, String contentType) { + this(length, modified, etag, sha256, bucket, key, contentType, Map.of(), Map.of(), null); + } + } record OpenObject(Metadata metadata, InputStream stream) implements AutoCloseable { public void close() throws IOException { stream.close(); } } @@ -15,12 +50,72 @@ interface ObjectStorage extends AutoCloseable { record ListPage(List objects, List prefixes, String nextKey, boolean truncated) { int keyCount() { return objects.size() + prefixes.size(); } } + record VersionEntry(String key, String versionId, long modified, boolean deleteMarker, + boolean latest, Metadata metadata) {} + record VersionPage(List entries, String nextKey, String nextVersionId, + boolean truncated) {} + record DeleteResult(String versionId, boolean deleteMarker) {} Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, - String checksum, boolean createOnly, String contentType) throws IOException; + String checksum, boolean createOnly, String contentType, + Map userMetadata, Map tags, + java.util.function.Supplier> checksums, + Map acl) throws IOException; + default Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, + String checksum, boolean createOnly, String contentType, + Map userMetadata, Map tags, + java.util.function.Supplier> checksums) throws IOException { + return put(bucket, key, input, length, expectedHash, checksum, createOnly, contentType, + userMetadata, tags, checksums, Map.of()); + } + default Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, + String checksum, boolean createOnly, String contentType, + Map userMetadata, Map tags) throws IOException { + return put(bucket, key, input, length, expectedHash, checksum, createOnly, contentType, + userMetadata, tags, Map::of); + } + default Metadata put(String bucket, String key, InputStream input, long length, String expectedHash, + String checksum, boolean createOnly, String contentType) throws IOException { + return put(bucket, key, input, length, expectedHash, checksum, createOnly, contentType, Map.of(), Map.of()); + } OpenObject open(String bucket, String key) throws IOException; + default OpenObject open(String bucket, String key, String versionId) throws IOException { + if (versionId == null) return open(bucket, key); + throw new StoreException(501, "NotImplemented", "Object versioning is unavailable"); + } + Map tags(String bucket, String key) throws IOException; + default Map tags(String bucket, String key, String versionId) throws IOException { + if (versionId == null) return tags(bucket, key); + throw new StoreException(501, "NotImplemented", "Versioned tagging is unavailable"); + } + void setTags(String bucket, String key, Map tags) throws IOException; + default void setTags(String bucket, String key, String versionId, + Map tags) throws IOException { + if (versionId == null) setTags(bucket, key, tags); + else throw new StoreException(501, "NotImplemented", "Versioned tagging is unavailable"); + } void delete(String bucket, String key) throws IOException; + default DeleteResult delete(String bucket, String key, String versionId) throws IOException { + if (versionId != null) throw new StoreException(501, "NotImplemented", "Object versioning is unavailable"); + delete(bucket, key); + return new DeleteResult(null, false); + } + default void setVersioning(String bucket, VersioningState state) throws IOException { + throw new StoreException(501, "NotImplemented", "Object versioning is unavailable"); + } + default VersionPage listVersions(String bucket, String prefix, String keyMarker, + String versionMarker, int maxKeys) throws IOException { + throw new StoreException(501, "NotImplemented", "Object versioning is unavailable"); + } ListPage list(String bucket, String prefix, String delimiter, int maxKeys, String after) throws IOException; + void ensureBucket(String bucket) throws IOException; + Bucket bucket(String bucket) throws IOException; + List buckets() throws IOException; + void createBucket(String bucket) throws IOException; + void deleteBucket(String bucket) throws IOException; + void setBucketAcl(String bucket, Map acl) throws IOException; + void setObjectAcl(String bucket, String key, String versionId, Map acl) throws IOException; + Limits limits(); default boolean ready() { return true; } void close() throws IOException; } diff --git a/src/cloud/lunarsky/store/SchemaMigrator.java b/src/cloud/lunarsky/store/SchemaMigrator.java index 09b3b00..d9c8c65 100644 --- a/src/cloud/lunarsky/store/SchemaMigrator.java +++ b/src/cloud/lunarsky/store/SchemaMigrator.java @@ -21,7 +21,7 @@ final class SchemaMigrator { result.next(); version = result.getInt(1); } - if (version > 4) throw new IOException("Metadata schema is newer than this ObjectStore build"); + if (version > 10) throw new IOException("Metadata schema is newer than this ObjectStore build"); if (version < 1) { statement.execute("CREATE TABLE IF NOT EXISTS cluster_usage (bucket text PRIMARY KEY, used_bytes bigint NOT NULL CHECK (used_bytes >= 0))"); statement.execute("CREATE TABLE IF NOT EXISTS cluster_objects (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, length bigint NOT NULL, modified bigint NOT NULL, etag text NOT NULL, sha256 bytea NOT NULL, content_type text NOT NULL, PRIMARY KEY (bucket, object_key))"); @@ -47,12 +47,69 @@ final class SchemaMigrator { statement.execute("CREATE TABLE cluster_gc_candidates (node_id uuid NOT NULL, segment_id uuid NOT NULL, observed_mtime bigint NOT NULL, first_seen bigint NOT NULL, PRIMARY KEY (node_id, segment_id))"); statement.execute("INSERT INTO cluster_schema_migrations VALUES (4)"); } + if (version < 5) { + statement.execute("ALTER TABLE cluster_objects ADD COLUMN user_metadata bytea"); + statement.execute("ALTER TABLE cluster_objects ADD COLUMN tags bytea"); + statement.execute("ALTER TABLE cluster_uploads ADD COLUMN user_metadata bytea"); + statement.execute("ALTER TABLE cluster_uploads ADD COLUMN tags bytea"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (5)"); + } + if (version < 6) { + statement.execute("CREATE TABLE cluster_buckets (name text PRIMARY KEY, created_at bigint NOT NULL)"); + statement.execute("INSERT INTO cluster_buckets SELECT DISTINCT bucket, " + + "CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS bigint) FROM cluster_usage"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (6)"); + } + if (version < 7) { + statement.execute("ALTER TABLE cluster_buckets ADD COLUMN versioning_state text NOT NULL " + + "DEFAULT 'NEVER' CHECK (versioning_state IN ('NEVER', 'ENABLED', 'SUSPENDED'))"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (7)"); + } + if (version < 8) { + statement.execute("CREATE TABLE cluster_object_versions (sequence bigint GENERATED ALWAYS AS IDENTITY, " + + "bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, version_id text NOT NULL, " + + "delete_marker boolean NOT NULL, generation uuid, length bigint, modified bigint NOT NULL, " + + "etag text, sha256 bytea, content_type text, user_metadata bytea, tags bytea, " + + "PRIMARY KEY (bucket, object_key, version_id), " + + "CHECK (delete_marker = (generation IS NULL)))"); + statement.execute("CREATE INDEX cluster_versions_order ON cluster_object_versions " + + "(bucket, object_key, sequence DESC)"); + statement.execute("CREATE TABLE cluster_object_heads (bucket text NOT NULL, " + + "object_key text COLLATE \"C\" NOT NULL, version_id text NOT NULL, " + + "PRIMARY KEY (bucket, object_key), " + + "FOREIGN KEY (bucket, object_key, version_id) REFERENCES cluster_object_versions " + + "(bucket, object_key, version_id) DEFERRABLE INITIALLY DEFERRED)"); + statement.execute("INSERT INTO cluster_object_versions " + + "(bucket, object_key, version_id, delete_marker, generation, length, modified, etag, " + + "sha256, content_type, user_metadata, tags) " + + "SELECT bucket, object_key, 'null', false, generation, length, modified, etag, sha256, " + + "content_type, user_metadata, tags FROM cluster_objects"); + statement.execute("INSERT INTO cluster_object_heads " + + "SELECT bucket, object_key, 'null' FROM cluster_objects"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (8)"); + } + if (version < 9) { + statement.execute("ALTER TABLE cluster_object_versions ADD COLUMN checksum_metadata bytea"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (9)"); + } + if (version < 10) { + statement.execute("ALTER TABLE cluster_buckets ADD COLUMN acl bytea"); + statement.execute("ALTER TABLE cluster_object_versions ADD COLUMN acl bytea"); + statement.execute("ALTER TABLE cluster_uploads ADD COLUMN acl bytea"); + statement.execute("INSERT INTO cluster_schema_migrations VALUES (10)"); + } statement.execute("INSERT INTO cluster_format SELECT 1, CASE WHEN EXISTS (SELECT 1 FROM cluster_segments WHERE replica_ids IS NULL) THEN 1 ELSE 2 END WHERE NOT EXISTS (SELECT 1 FROM cluster_format)"); } try (PreparedStatement insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, 0) ON CONFLICT DO NOTHING")) { insert.setString(1, bucket); insert.executeUpdate(); } + try (PreparedStatement insert = connection.prepareStatement( + "INSERT INTO cluster_buckets (name, created_at) VALUES (?, ?) ON CONFLICT DO NOTHING")) { + insert.setString(1, bucket); + insert.setLong(2, System.currentTimeMillis()); + insert.executeUpdate(); + } int format; try (Statement statement = connection.createStatement(); ResultSet result = statement.executeQuery("SELECT version FROM cluster_format WHERE singleton=1")) { diff --git a/src/cloud/lunarsky/store/SigV4.java b/src/cloud/lunarsky/store/SigV4.java index c78eaf7..42f6828 100644 --- a/src/cloud/lunarsky/store/SigV4.java +++ b/src/cloud/lunarsky/store/SigV4.java @@ -12,39 +12,124 @@ import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.HexFormat; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; final class SigV4 { + record Verified(String payload, String applicationQuery, byte[] signingKey, + String date, String scope, String signature, String principal) { + Verified(String payload, String applicationQuery, byte[] signingKey, + String date, String scope, String signature) { + this(payload, applicationQuery, signingKey, date, scope, signature, null); + } + boolean streaming() { return payload.startsWith("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"); } + } private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC); private static final Pattern HEX = Pattern.compile("[0-9a-f]{64}"); - private final String accessKey, secretKey, region; + private final Map identities; + private final String root; + private final String region; private final Clock clock; SigV4(String accessKey, String secretKey, String region, Clock clock) { - this.accessKey = accessKey; - this.secretKey = secretKey; + this(Map.of(accessKey, secretKey), accessKey, region, clock); + } + + SigV4(Map identities, String root, String region, Clock clock) { + this.identities = Map.copyOf(identities); + if (!this.identities.containsKey(root)) throw new IllegalArgumentException("Missing root identity"); + this.root = root; this.region = region; this.clock = clock; } + String root() { return root; } + Set identities() { return identities.keySet(); } + String verify(String method, URI uri, Headers headers) { + return verifyRequest(method, uri, headers).payload(); + } + + Verified verifyRequest(String method, URI uri, Headers headers) { + if (hasPresignedQuery(uri.getRawQuery())) return verifyPresigned(method, uri, headers); Map fields = authorizationFields(headers); String[] credential = credentialScope(fields.get("Credential")); String date = signingDate(headers, credential[1]); String payload = payloadHash(headers); String signedHeaders = fields.get("SignedHeaders"); - String canonicalHeaders = canonicalHeaders(headers, signedHeaders); + String canonicalHeaders = canonicalHeaders(headers, signedHeaders, + Set.of("host", "x-amz-date", "x-amz-content-sha256")); String canonical = method + "\n" + encode(decode(uri.getRawPath()), true) + "\n" + canonicalQuery(uri.getRawQuery()) + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payload; String scope = String.join("/", Arrays.copyOfRange(credential, 1, 5)); String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + hex(hash(canonical.getBytes(StandardCharsets.UTF_8))); - byte[] signingKey = signingKey(secretKey, credential[1], region); + byte[] signingKey = signingKey(secret(credential[0]), credential[1], region); String signature = fields.get("Signature"); if (!HEX.matcher(signature).matches() || !MessageDigest.isEqual(hmac(signingKey, toSign), HexFormat.of().parseHex(signature))) denied("Signature mismatch"); - return payload; + return new Verified(payload, uri.getRawQuery(), signingKey, date, scope, signature, credential[0]); + } + + private Verified verifyPresigned(String method, URI uri, Headers headers) { + if (headers.containsKey("authorization")) denied("Use one authentication method"); + Map fields = new TreeMap<>(); + StringBuilder application = new StringBuilder(); + StringBuilder signed = new StringBuilder(); + for (String part : uri.getRawQuery().split("&", -1)) { + String[] pair = part.split("=", 2); + String name = decode(pair[0]); + String value = decode(pair.length == 2 ? pair[1] : ""); + if (name.startsWith("X-Amz-")) { + if (fields.put(name, value) != null) denied("Duplicate presigned parameter"); + if (!name.equals("X-Amz-Signature")) appendQuery(signed, part); + } else { + appendQuery(application, part); + appendQuery(signed, part); + } + } + if (!fields.keySet().equals(Set.of("X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date", + "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Signature")) || + !"AWS4-HMAC-SHA256".equals(fields.get("X-Amz-Algorithm"))) + denied("Invalid presigned parameters"); + String[] credential = credentialScope(fields.get("X-Amz-Credential")); + String date = fields.get("X-Amz-Date"); + if (!date.matches("[0-9]{8}T[0-9]{6}Z") || !date.startsWith(credential[1])) + denied("Invalid signing date"); + long expires; + try { expires = Long.parseLong(fields.get("X-Amz-Expires")); } + catch (NumberFormatException error) { denied("Invalid presigned expiry"); return null; } + if (expires < 1 || expires > 604800) denied("Invalid presigned expiry"); + try { + Instant start = Instant.from(DATE.parse(date)); + Instant now = clock.instant(); + if (now.isBefore(start.minus(Duration.ofMinutes(5))) || now.isAfter(start.plusSeconds(expires))) + denied("Presigned URL has expired or is not yet valid"); + } catch (java.time.DateTimeException error) { denied("Invalid signing date"); } + String signedHeaders = fields.get("X-Amz-SignedHeaders"); + String canonicalHeaders = canonicalHeaders(headers, signedHeaders, Set.of("host")); + String scope = String.join("/", Arrays.copyOfRange(credential, 1, 5)); + String canonical = method + "\n" + encode(decode(uri.getRawPath()), true) + "\n" + + canonicalQuery(signed.toString()) + "\n" + canonicalHeaders + "\n" + + signedHeaders + "\nUNSIGNED-PAYLOAD"; + String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + + hex(hash(canonical.getBytes(StandardCharsets.UTF_8))); + String signature = fields.get("X-Amz-Signature"); + byte[] key = signingKey(secret(credential[0]), credential[1], region); + if (!HEX.matcher(signature).matches() || + !MessageDigest.isEqual(hmac(key, toSign), HexFormat.of().parseHex(signature))) + denied("Signature mismatch"); + return new Verified("UNSIGNED-PAYLOAD", application.toString(), key, date, scope, signature, credential[0]); + } + + private static boolean hasPresignedQuery(String raw) { + return raw != null && (raw.startsWith("X-Amz-Algorithm=") || raw.contains("&X-Amz-Algorithm=")); + } + + private static void appendQuery(StringBuilder target, String part) { + if (!target.isEmpty()) target.append('&'); + target.append(part); } private static Map authorizationFields(Headers headers) { @@ -61,11 +146,17 @@ final class SigV4 { private String[] credentialScope(String value) { String[] credential = value.split("/", -1); - if (credential.length != 5 || !credential[0].equals(accessKey) || !credential[2].equals(region) + if (credential.length != 5 || !identities.containsKey(credential[0]) || !credential[2].equals(region) || !credential[3].equals("s3") || !credential[4].equals("aws4_request")) denied("Invalid credential scope"); return credential; } + private String secret(String accessKey) { + String secret = identities.get(accessKey); + if (secret == null) denied("Invalid credential scope"); + return secret; + } + private String signingDate(Headers headers, String credentialDate) { String date = single(headers, "x-amz-date"); if (date == null || !credentialDate.matches("[0-9]{8}") || !date.matches("[0-9]{8}T[0-9]{6}Z") || !date.startsWith(credentialDate)) denied("Invalid signing date"); @@ -79,17 +170,19 @@ final class SigV4 { private static String payloadHash(Headers headers) { String payload = single(headers, "x-amz-content-sha256"); - if (payload == null || !HEX.matcher(payload).matches()) - throw new StoreException(400, "NotImplemented", "A hexadecimal SHA-256 payload hash is required; unsigned and chunk-signed payloads are unsupported"); + if (payload == null || !(HEX.matcher(payload).matches() || + payload.equals("STREAMING-AWS4-HMAC-SHA256-PAYLOAD") || + payload.equals("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"))) + throw new StoreException(400, "NotImplemented", "Unsupported SHA-256 payload mode"); if (headers.containsKey("x-amz-security-token")) denied("Temporary credentials are unsupported"); return payload; } - private static String canonicalHeaders(Headers headers, String signedHeaders) { + private static String canonicalHeaders(Headers headers, String signedHeaders, Set required) { String[] names = signedHeaders.split(";", -1); if (names.length > 32 || !signedHeaders.equals(String.join(";", Arrays.stream(names).distinct().sorted().toList()))) denied("Signed headers must be unique and sorted"); var namesSet = java.util.Set.copyOf(Arrays.asList(names)); - if (!namesSet.containsAll(java.util.Set.of("host", "x-amz-date", "x-amz-content-sha256"))) denied("Missing signed headers"); + if (!namesSet.containsAll(required)) denied("Missing signed headers"); for (String key : headers.keySet()) { String lower = key.toLowerCase(java.util.Locale.ROOT); if (lower.startsWith("x-amz-") && !namesSet.contains(lower)) denied("Unsigned Amazon header"); diff --git a/src/cloud/lunarsky/store/StoreException.java b/src/cloud/lunarsky/store/StoreException.java index 8905458..276dc28 100644 --- a/src/cloud/lunarsky/store/StoreException.java +++ b/src/cloud/lunarsky/store/StoreException.java @@ -3,6 +3,9 @@ package cloud.lunarsky.store; final class StoreException extends RuntimeException { final int status; final String code; + final String versionId; + final long modified; + final boolean deleteMarker; StoreException(int status, String code, String message) { this(status, code, message, null); } @@ -10,5 +13,20 @@ final class StoreException extends RuntimeException { super(message, cause); this.status = status; this.code = code; + this.versionId = null; + this.modified = -1; + this.deleteMarker = false; + } + private StoreException(int status, String code, String message, String versionId, long modified) { + super(message); + this.status = status; + this.code = code; + this.versionId = versionId; + this.modified = modified; + this.deleteMarker = true; + } + static StoreException deletedVersion(String versionId, long modified, boolean explicit) { + return new StoreException(explicit ? 405 : 404, explicit ? "MethodNotAllowed" : "NoSuchKey", + "Object is deleted", versionId, modified); } } diff --git a/src/cloud/lunarsky/store/UploadChecksums.java b/src/cloud/lunarsky/store/UploadChecksums.java index d5eb52d..2effe8d 100644 --- a/src/cloud/lunarsky/store/UploadChecksums.java +++ b/src/cloud/lunarsky/store/UploadChecksums.java @@ -41,8 +41,16 @@ final class UploadChecksums { encoded = SigV4.single(headers, name); } String selected = SigV4.single(headers, "x-amz-sdk-checksum-algorithm"); - if (selected != null && (algorithm == null || !selected.equals(algorithm.name()))) - throw new StoreException(400, "InvalidRequest", "Checksum algorithm and value must match"); + if (selected != null) { + String trailer = SigV4.single(headers, "x-amz-trailer"); + if (algorithm == null && trailer != null) { + Algorithm declared = algorithm(trailer); + if (!selected.equals(declared.name())) + throw new StoreException(400, "InvalidRequest", "Checksum algorithm and trailer must match"); + } else if (algorithm == null || !selected.equals(algorithm.name())) { + throw new StoreException(400, "InvalidRequest", "Checksum algorithm and value must match"); + } + } byte[] expected = algorithm == null ? null : decode(encoded, algorithm.length()); return new UploadChecksums(contentMd5, algorithm, expected, encoded); } @@ -51,6 +59,10 @@ final class UploadChecksums { return switch (header) { case "x-amz-checksum-crc32" -> new Algorithm("CRC32", header, 4); case "x-amz-checksum-crc32c" -> new Algorithm("CRC32C", header, 4); + case "x-amz-checksum-crc64nvme" -> new Algorithm("CRC64NVME", header, 8); + case "x-amz-checksum-xxhash64" -> new Algorithm("XXHASH64", header, 8); + case "x-amz-checksum-xxhash3" -> new Algorithm("XXHASH3", header, 8); + case "x-amz-checksum-xxhash128" -> new Algorithm("XXHASH128", header, 16); case "x-amz-checksum-sha1" -> new Algorithm("SHA1", header, 20); case "x-amz-checksum-sha256" -> new Algorithm("SHA256", header, 32); case "x-amz-checksum-sha512" -> new Algorithm("SHA512", header, 64); @@ -71,6 +83,10 @@ final class UploadChecksums { return algorithm != null && algorithm.name().equals("SHA256") ? encoded : null; } + java.util.Map metadata() { + return algorithm == null ? java.util.Map.of() : java.util.Map.of(algorithm.header(), encoded); + } + void response(Headers headers) { if (algorithm != null) headers.set(algorithm.header(), encoded); } @@ -91,8 +107,11 @@ final class UploadChecksums { private final Checksum crc = algorithm == null ? null : switch (algorithm.name()) { case "CRC32" -> new CRC32(); case "CRC32C" -> new CRC32C(); + case "CRC64NVME" -> new Crc64Nvme(); default -> null; }; + private final XxHashes xxhash = algorithm != null && algorithm.name().startsWith("XXHASH") + ? new XxHashes(algorithm.name()) : null; private boolean checked; private VerifiedInput(InputStream input) { super(input); } @@ -115,6 +134,7 @@ final class UploadChecksums { if (md5 != null) md5.update(bytes, offset, length); if (hash != null) hash.update(bytes, offset, length); if (crc != null) crc.update(bytes, offset, length); + if (xxhash != null) xxhash.update(bytes, offset, length); } private void verify() { @@ -127,9 +147,13 @@ final class UploadChecksums { byte[] actual; if (crc != null) { long value = crc.getValue(); - actual = new byte[]{(byte) (value >>> 24), (byte) (value >>> 16), - (byte) (value >>> 8), (byte) value}; + actual = new byte[algorithm.length()]; + for (int i = actual.length - 1; i >= 0; i--) { + actual[i] = (byte) value; + value >>>= 8; + } } else if (algorithm.name().equals("MD5")) actual = actualMd5; + else if (xxhash != null) actual = xxhash.digest(); else actual = hash.digest(); if (!MessageDigest.isEqual(expected, actual)) throw new StoreException(400, "BadDigest", algorithm.name() + " checksum mismatch"); diff --git a/src/cloud/lunarsky/store/Version.java b/src/cloud/lunarsky/store/Version.java index 4e09bb2..2d9dea2 100644 --- a/src/cloud/lunarsky/store/Version.java +++ b/src/cloud/lunarsky/store/Version.java @@ -1,7 +1,7 @@ package cloud.lunarsky.store; final class Version { - static final String VALUE = "0.0.4"; + static final String VALUE = "0.0.8"; private Version() {} } diff --git a/src/cloud/lunarsky/store/XxHashes.java b/src/cloud/lunarsky/store/XxHashes.java new file mode 100644 index 0000000..f70f47e --- /dev/null +++ b/src/cloud/lunarsky/store/XxHashes.java @@ -0,0 +1,32 @@ +package cloud.lunarsky.store; + +import com.dynatrace.hash4j.hashing.HashStream64; +import com.dynatrace.hash4j.hashing.HashStream128; +import com.dynatrace.hash4j.hashing.Hashing; +import java.nio.ByteBuffer; + +final class XxHashes { + private final HashStream64 stream; + + XxHashes(String algorithm) { + stream = switch (algorithm) { + case "XXHASH64" -> Hashing.xxh64().hashStream(); + case "XXHASH3" -> Hashing.xxh3_64().hashStream(); + case "XXHASH128" -> Hashing.xxh3_128().hashStream(); + default -> throw new IllegalArgumentException(algorithm); + }; + } + + void update(byte[] bytes, int offset, int length) { + stream.putBytes(bytes, offset, length); + } + + byte[] digest() { + if (stream instanceof HashStream128 wide) { + var value = wide.get(); + return ByteBuffer.allocate(16).putLong(value.getMostSignificantBits()) + .putLong(value.getLeastSignificantBits()).array(); + } + return ByteBuffer.allocate(8).putLong(stream.getAsLong()).array(); + } +} diff --git a/test/cloud/lunarsky/store/CliTest.java b/test/cloud/lunarsky/store/CliTest.java index 90907ff..f6e551c 100644 --- a/test/cloud/lunarsky/store/CliTest.java +++ b/test/cloud/lunarsky/store/CliTest.java @@ -44,6 +44,21 @@ public final class CliTest { int status = Cli.run(new String[]{"verify"}, root, new PrintStream(output), new PrintStream(output)); if (status != 1 || !output.toString().contains("Object checksum mismatch")) throw new AssertionError("CLI missed corrupted payload"); + Path versionRoot = root.resolve("versioned"); + try (var store = new DiskStore(versionRoot, 100, 1000)) { + store.createBucket("versioned-bucket"); + store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.ENABLED); + for (byte[] body : new byte[][]{"first".getBytes(StandardCharsets.UTF_8), + "second".getBytes(StandardCharsets.UTF_8)}) { + store.put("versioned-bucket", "example", new ByteArrayInputStream(body), body.length, + SigV4.hex(SigV4.hash(body)), null, false, "text/plain"); + } + output.reset(); + status = Cli.run(new String[]{"verify"}, versionRoot, + new PrintStream(output), new PrintStream(output)); + if (status != 0 || !output.toString().contains("verified_objects=2")) + throw new AssertionError("CLI did not inspect retained versions"); + } System.out.println("CLI tests passed: version, live status, verification, corruption exit code"); } finally { try (var paths = Files.walk(root)) { diff --git a/test/cloud/lunarsky/store/ClientLimitsTest.java b/test/cloud/lunarsky/store/ClientLimitsTest.java new file mode 100644 index 0000000..fb0d9d8 --- /dev/null +++ b/test/cloud/lunarsky/store/ClientLimitsTest.java @@ -0,0 +1,98 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.HttpServer; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.Executors; + +public final class ClientLimitsTest { + private static final String ACCESS = "TESTACCESSKEY123"; + private static final String SECRET = "test-secret-key-that-is-at-least-32-characters"; + + private static HttpResponse get(HttpClient client, String base, String path, String ip) throws Exception { + var request = HttpRequest.newBuilder(URI.create(base + path)); + if (ip != null) request.header("X-Real-IP", ip); + return client.send(request.GET().build(), HttpResponse.BodyHandlers.ofString()); + } + + private static void status(int wanted, HttpResponse response) { + if (response.statusCode() != wanted) + throw new AssertionError("Expected " + wanted + ", got " + response.statusCode() + ": " + response.body()); + } + + private static void exercise(Map configuration, boolean trusted) throws Exception { + Path root = Files.createTempDirectory("client-limits-test-"); + var executor = Executors.newVirtualThreadPerTaskExecutor(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16); + DiskStore store = new DiskStore(root, 1024, 4096); + try { + var app = new Main(store, new MultipartStore(store), + new SigV4(Map.of(ACCESS, SECRET), ACCESS, "us-east-1", Clock.systemUTC()), + "objects", ClientLimits.fromEnvironment(configuration)); + server.setExecutor(executor); + server.createContext("/", app::handle); + server.start(); + String base = "http://127.0.0.1:" + server.getAddress().getPort(); + HttpClient client = HttpClient.newHttpClient(); + if (trusted) { + status(200, get(client, base, "/health", null)); + status(400, get(client, base, "/ready", null)); + status(400, get(client, base, "/objects/example", null)); + status(400, get(client, base, "/health", "not-an-ip")); + status(400, client.send(HttpRequest.newBuilder(URI.create(base + "/health")) + .header("X-Real-IP", "192.0.2.1") + .header("X-Real-IP", "192.0.2.2") + .GET().build(), HttpResponse.BodyHandlers.ofString())); + status(200, get(client, base, "/health", "192.0.2.1")); + var limited = get(client, base, "/health", "192.0.2.1"); + status(503, limited); + if (!"1".equals(limited.headers().firstValue("Retry-After").orElse(null))) + throw new AssertionError("SlowDown response lacks Retry-After"); + status(200, get(client, base, "/health", "192.0.2.2")); + } else if (configuration.containsKey("PUBLIC_BYTES_PER_SECOND")) { + long start = System.nanoTime(); + byte[] upload = new byte[128]; + status(200, client.send(HttpTest.signedUri(URI.create(base + "/objects/bandwidth"), + "PUT", upload, Map.of()), HttpResponse.BodyHandlers.ofString())); + if (System.nanoTime() - start < 800_000_000L) + throw new AssertionError("Upload bytes were not paced"); + start = System.nanoTime(); + status(200, get(client, base, "/health", "192.0.2.1")); + status(200, get(client, base, "/health", "192.0.2.1")); + if (System.nanoTime() - start < 250_000_000L) + throw new AssertionError("Responses were not paced by the shared byte budget"); + } else { + status(200, get(client, base, "/health", "192.0.2.1")); + status(503, get(client, base, "/health", "192.0.2.2")); + } + } finally { + server.stop(0); + executor.close(); + store.close(); + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path); + } + } + } + + public static void main(String[] args) throws Exception { + var disabled = ClientLimits.fromEnvironment(Map.of()); + if (disabled == null) throw new AssertionError("Disabled configuration missing"); + try { ClientLimits.fromEnvironment(Map.of("PUBLIC_TRUSTED_PROXY_IPS", "127.0.0.1")); + throw new AssertionError("Proxy trust accepted without limits"); + } catch (IllegalArgumentException expected) { } + exercise(Map.of("PUBLIC_REQUESTS_PER_SECOND", "1", "PUBLIC_REQUEST_BURST", "1", + "PUBLIC_TRUSTED_PROXY_IPS", "127.0.0.1"), true); + exercise(Map.of("PUBLIC_REQUESTS_PER_SECOND", "1", "PUBLIC_REQUEST_BURST", "1"), false); + exercise(Map.of("PUBLIC_BYTES_PER_SECOND", "64", "PUBLIC_BYTE_BURST", "64"), false); + System.out.println("Client limit tests passed"); + } +} diff --git a/test/cloud/lunarsky/store/ClusterNodeTest.java b/test/cloud/lunarsky/store/ClusterNodeTest.java index 0b6ea2e..8286035 100644 --- a/test/cloud/lunarsky/store/ClusterNodeTest.java +++ b/test/cloud/lunarsky/store/ClusterNodeTest.java @@ -138,7 +138,25 @@ public final class ClusterNodeTest { throw new AssertionError("Oversized segment response was accepted"); } catch (IOException expected) { } } finally { oversized.stop(0); } - System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup"); + URI stopped = URI.create("http://127.0.0.1:" + oversized.getAddress().getPort()); + NodeClient readOnly = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, stopped)), token, null); + try { + readOnly.get(0, id, value.length, SigV4.hash(value)); + throw new AssertionError("Read-only access did not detect a stopped node"); + } catch (IOException expected) { + require(expected.getMessage().contains("temporarily unreachable"), + "Read-only access did not use the short health probe"); + } + NodeClient offline = new NodeClient(List.of(new NodeClient.Node(nodeId, hostId, stopped)), token, null); + require(!offline.availableHostsAtLeast(1, false), "Stopped node was reported healthy"); + try { + offline.get(0, id, value.length, SigV4.hash(value)); + throw new AssertionError("Stopped node was read after a failed health check"); + } catch (IOException expected) { + require(expected.getMessage().contains("temporarily unreachable"), + "Read did not skip a recently failed node"); + } + System.out.println("Cluster node tests passed: lock, authenticated roundtrip, checksums, restart cleanup, outage fallback"); } private static void require(boolean condition, String message) { if (!condition) throw new AssertionError(message); diff --git a/test/cloud/lunarsky/store/HttpTest.java b/test/cloud/lunarsky/store/HttpTest.java index 69c4ddf..1e4a804 100644 --- a/test/cloud/lunarsky/store/HttpTest.java +++ b/test/cloud/lunarsky/store/HttpTest.java @@ -25,6 +25,8 @@ import java.util.zip.Checksum; public final class HttpTest { private static final String ACCESS = "TESTACCESSKEY123"; private static final String SECRET = "test-secret-key-that-is-at-least-32-characters"; + private static final String SECONDARY = "SECONDARYKEY1234"; + private static final String SECONDARY_SECRET = "secondary-secret-key-that-is-at-least-32-characters"; private static final String REGION = "us-east-1"; private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC); @@ -33,7 +35,12 @@ public final class HttpTest { return signedUri(URI.create(base + "/objects/" + SigV4.encode(key, true)), method, body, Map.of()); } - private static HttpRequest signedUri(URI uri, String method, byte[] body, Map extra) { + static HttpRequest signedUri(URI uri, String method, byte[] body, Map extra) { + return signedUriAs(uri, method, body, extra, ACCESS, SECRET); + } + + private static HttpRequest signedUriAs(URI uri, String method, byte[] body, Map extra, + String access, String secret) { String host = uri.getAuthority(); String date = DATE.format(Instant.now()); String hash = SigV4.hex(SigV4.hash(body)); @@ -50,11 +57,11 @@ public final class HttpTest { String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + SigV4.hex(SigV4.hash(canonical.toString().getBytes(StandardCharsets.UTF_8))); String signature = SigV4.hex(SigV4.hmac( - SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign)); + SigV4.signingKey(secret, date.substring(0, 8), REGION), toSign)); HttpRequest.Builder request = HttpRequest.newBuilder(uri) .header("x-amz-date", date) .header("x-amz-content-sha256", hash) - .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + .header("authorization", "AWS4-HMAC-SHA256 Credential=" + access + "/" + scope + ",SignedHeaders=" + names + ",Signature=" + signature); extra.forEach(request::header); return request.method(method, body.length == 0 @@ -63,10 +70,275 @@ public final class HttpTest { .build(); } - private static void status(int expected, HttpResponse response) { + private static URI presignedUri(URI uri, String method, int expires) { + String date = DATE.format(Instant.now()); + String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; + String query = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=" + + SigV4.encode(ACCESS + "/" + scope, false) + "&X-Amz-Date=" + date + + "&X-Amz-Expires=" + expires + "&X-Amz-SignedHeaders=host"; + String canonical = method + "\n" + uri.getRawPath() + "\n" + + SigV4.canonicalQuery(query) + "\nhost:" + uri.getAuthority() + + "\n\nhost\nUNSIGNED-PAYLOAD"; + String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n" + + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8))); + String signature = SigV4.hex(SigV4.hmac(SigV4.signingKey(SECRET, + date.substring(0, 8), REGION), toSign)); + return URI.create(uri + "?" + query + "&X-Amz-Signature=" + signature); + } + + private static void testPresigned(HttpClient client, String base) throws Exception { + URI object = URI.create(base + "/objects/presigned-test"); + byte[] body = "presigned upload".getBytes(StandardCharsets.UTF_8); + status(200, client.send(HttpRequest.newBuilder(presignedUri(object, "PUT", 60)) + .PUT(HttpRequest.BodyPublishers.ofByteArray(body)).build(), + HttpResponse.BodyHandlers.ofByteArray())); + var read = client.send(HttpRequest.newBuilder(presignedUri(object, "GET", 60)).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, read); + if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Presigned object mismatch"); + URI tampered = URI.create(presignedUri(object, "GET", 60).toString().replace("presigned-test", "different")); + status(403, client.send(HttpRequest.newBuilder(tampered).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + status(204, client.send(HttpRequest.newBuilder(presignedUri(object, "DELETE", 60)) + .DELETE().build(), HttpResponse.BodyHandlers.ofByteArray())); + } + + private static void testAcl(HttpClient client, String base) throws Exception { + byte[] body = "private object".getBytes(StandardCharsets.UTF_8); + URI object = URI.create(base + "/objects/acl-test"); + URI objectAcl = URI.create(object + "?acl"); + status(200, client.send(signedUri(object, "PUT", body, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(), + SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(signedUriAs(object, "GET", new byte[0], Map.of(), + SECONDARY, SECRET), HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(signedUriAs(URI.create(base + "/_objectstore/capabilities"), + "GET", new byte[0], Map.of(), SECONDARY, SECONDARY_SECRET), + HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(HttpRequest.newBuilder(object).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(objectAcl, "PUT", new byte[0], + Map.of("x-amz-grant-read", "id=\"" + SECONDARY + "\"")), + HttpResponse.BodyHandlers.ofByteArray())); + var read = client.send(signedUriAs(object, "GET", new byte[0], Map.of(), + SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray()); + status(200, read); + if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("ACL read mismatch"); + status(403, client.send(signedUriAs(object, "DELETE", new byte[0], Map.of(), + SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); + var acl = client.send(signedUri(objectAcl, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofString()); + status(200, acl); + if (!acl.body().contains(SECONDARY)) throw new AssertionError("Object ACL grant missing"); + status(200, client.send(signedUri(URI.create(object + "?acl&x-id=GetObjectAcl"), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(objectAcl, "PUT", + acl.body().getBytes(StandardCharsets.UTF_8), Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + + URI bucketAcl = URI.create(base + "/objects?acl"); + status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0], + Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(HttpRequest.newBuilder( + URI.create(base + "/objects?list-type=2")).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(HttpRequest.newBuilder( + URI.create(base + "/objects?uploads")).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(bucketAcl, "PUT", new byte[0], + Map.of("x-amz-grant-write", "id=\"" + SECONDARY + "\"")), + HttpResponse.BodyHandlers.ofByteArray())); + URI uploaded = URI.create(base + "/objects/secondary-upload"); + status(200, client.send(signedUriAs(uploaded, "PUT", body, Map.of(), + SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(signedUriAs(uploaded, "GET", new byte[0], Map.of(), + SECONDARY, SECONDARY_SECRET), HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(uploaded, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(object, "PUT", body, + Map.of("x-amz-acl", "public-read")), HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(HttpRequest.newBuilder(object).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(HttpRequest.newBuilder(object).DELETE().build(), + HttpResponse.BodyHandlers.ofByteArray())); + + URI historyBucket = URI.create(base + "/acl-history"); + status(200, client.send(signedUri(historyBucket, "PUT", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + byte[] versioning = "Enabled" + .getBytes(StandardCharsets.UTF_8); + status(200, client.send(signedUri(URI.create(historyBucket + "?versioning"), + "PUT", versioning, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + URI versioned = URI.create(historyBucket + "/versioned"); + var first = client.send(signedUri(versioned, "PUT", body, Map.of("x-amz-acl", "public-read")), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, first); + String oldVersion = first.headers().firstValue("x-amz-version-id").orElseThrow(); + status(200, client.send(signedUri(versioned, "PUT", body, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(HttpRequest.newBuilder(versioned).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + URI historical = URI.create(versioned + "?versionId=" + oldVersion); + status(200, client.send(HttpRequest.newBuilder(historical).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + URI historicalAcl = URI.create(historical + "&acl"); + status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0], + Map.of("x-amz-grant-write-acp", "id=\"" + SECONDARY + "\"")), + HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUriAs(historicalAcl, "PUT", new byte[0], + Map.of("x-amz-acl", "public-read"), SECONDARY, SECONDARY_SECRET), + HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(HttpRequest.newBuilder(historical).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(historicalAcl, "PUT", new byte[0], + Map.of("x-amz-acl", "private")), HttpResponse.BodyHandlers.ofByteArray())); + status(403, client.send(HttpRequest.newBuilder(historical).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + } + + private static void testStreaming(HttpClient client, String base) throws Exception { + URI object = URI.create(base + "/objects/streaming-test"); + byte[] body = "verified streaming payload".getBytes(StandardCharsets.UTF_8); + String date = DATE.format(Instant.now()); + String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; + String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length"; + String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" + + "host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode + + "\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length + + "\n\n" + names + "\n" + mode; + byte[] signingKey = SigV4.signingKey(SECRET, date.substring(0, 8), REGION); + String seed = SigV4.hex(SigV4.hmac(signingKey, "AWS4-HMAC-SHA256\n" + date + "\n" + + scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8))))); + String previous = seed; + var encoded = new java.io.ByteArrayOutputStream(); + for (byte[] chunk : new byte[][]{body, new byte[0]}) { + String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" + + previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" + + SigV4.hex(SigV4.hash(chunk)); + previous = SigV4.hex(SigV4.hmac(signingKey, toSign)); + encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n") + .getBytes(StandardCharsets.US_ASCII)); + encoded.write(chunk); + encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII)); + } + HttpRequest.Builder request = HttpRequest.newBuilder(object) + .header("content-encoding", "aws-chunked") + .header("x-amz-content-sha256", mode) + .header("x-amz-date", date) + .header("x-amz-decoded-content-length", Integer.toString(body.length)) + .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope + + ",SignedHeaders=" + names + ",Signature=" + seed); + status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(encoded.toByteArray())) + .build(), HttpResponse.BodyHandlers.ofByteArray())); + var read = client.send(signedUri(object, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, read); + if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Streaming upload mismatch"); + status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + } + + private static void testStreamingTrailer(HttpClient client, String base) throws Exception { + URI object = URI.create(base + "/objects/streaming-trailer-test"); + byte[] body = "verified trailer payload".getBytes(StandardCharsets.UTF_8); + String date = DATE.format(Instant.now()); + String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request"; + String mode = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; + String trailerName = "x-amz-checksum-xxhash3"; + String checksum = encodedChecksum("XXHASH3", body); + String names = "content-encoding;host;x-amz-content-sha256;x-amz-date;" + + "x-amz-decoded-content-length;x-amz-sdk-checksum-algorithm;x-amz-trailer"; + String canonical = "PUT\n" + object.getRawPath() + "\n\ncontent-encoding:aws-chunked\n" + + "host:" + object.getAuthority() + "\nx-amz-content-sha256:" + mode + + "\nx-amz-date:" + date + "\nx-amz-decoded-content-length:" + body.length + + "\nx-amz-sdk-checksum-algorithm:XXHASH3\nx-amz-trailer:" + trailerName + + "\n\n" + names + "\n" + mode; + byte[] key = SigV4.signingKey(SECRET, date.substring(0, 8), REGION); + String seed = SigV4.hex(SigV4.hmac(key, "AWS4-HMAC-SHA256\n" + date + "\n" + + scope + "\n" + SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8))))); + String previous = seed; + var encoded = new java.io.ByteArrayOutputStream(); + for (byte[] chunk : new byte[][]{body, new byte[0]}) { + String toSign = "AWS4-HMAC-SHA256-PAYLOAD\n" + date + "\n" + scope + "\n" + + previous + "\n" + SigV4.hex(SigV4.hash(new byte[0])) + "\n" + + SigV4.hex(SigV4.hash(chunk)); + previous = SigV4.hex(SigV4.hmac(key, toSign)); + encoded.write((Integer.toHexString(chunk.length) + ";chunk-signature=" + previous + "\r\n") + .getBytes(StandardCharsets.US_ASCII)); + encoded.write(chunk); + if (chunk.length != 0) encoded.write("\r\n".getBytes(StandardCharsets.US_ASCII)); + } + encoded.write((trailerName + ":" + checksum + "\r\n").getBytes(StandardCharsets.US_ASCII)); + String trailerToSign = "AWS4-HMAC-SHA256-TRAILER\n" + date + "\n" + scope + "\n" + + previous + "\n" + SigV4.hex(SigV4.hash((trailerName + ":" + checksum + "\n") + .getBytes(StandardCharsets.UTF_8))); + encoded.write(("x-amz-trailer-signature=" + SigV4.hex(SigV4.hmac(key, trailerToSign)) + + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII)); + byte[] upload = encoded.toByteArray(); + HttpRequest.Builder request = HttpRequest.newBuilder(object) + .header("content-encoding", "aws-chunked") + .header("x-amz-content-sha256", mode) + .header("x-amz-date", date) + .header("x-amz-decoded-content-length", Integer.toString(body.length)) + .header("x-amz-sdk-checksum-algorithm", "XXHASH3") + .header("x-amz-trailer", trailerName) + .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope + + ",SignedHeaders=" + names + ",Signature=" + seed); + status(200, client.send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(upload)).build(), + HttpResponse.BodyHandlers.ofByteArray())); + var head = client.send(signedUri(object, "HEAD", new byte[0], + Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding()); + status(200, head); + if (!checksum.equals(head.headers().firstValue(trailerName).orElse(""))) + throw new AssertionError("Signed checksum trailer was not persisted"); + byte[] tampered = upload.clone(); + tampered[upload.length - 10] ^= 1; + status(400, client.send(HttpRequest.newBuilder(object) + .header("content-encoding", "aws-chunked") + .header("x-amz-content-sha256", mode) + .header("x-amz-date", date) + .header("x-amz-decoded-content-length", Integer.toString(body.length)) + .header("x-amz-sdk-checksum-algorithm", "XXHASH3") + .header("x-amz-trailer", trailerName) + .header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/" + scope + + ",SignedHeaders=" + names + ",Signature=" + seed) + .PUT(HttpRequest.BodyPublishers.ofByteArray(tampered)).build(), + HttpResponse.BodyHandlers.ofByteArray())); + status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + } + + private static void testCapabilities(HttpClient client, String base) throws Exception { + URI uri = URI.create(base + "/_objectstore/capabilities"); + status(403, client.send(HttpRequest.newBuilder(uri).GET().build(), + HttpResponse.BodyHandlers.ofByteArray())); + var response = client.send(signedUri(uri, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofString()); + status(200, response); + String compact = response.body().replaceAll("\\s+", ""); + if (!compact.contains("\"schemaVersion\":1") || + !compact.contains("\"service\":\"lunarsky-objectstore\"") || + !compact.contains("\"serviceVersion\":\"" + Version.VALUE + "\"") || + !compact.contains("\"storageMode\":\"disk\"") || + !compact.contains("\"ListParts\"") || + !compact.contains("\"maxObjectBytes\":1024") || + !compact.contains("\"maxTotalBytes\":4096") || + !compact.contains("\"maxParts\":10000")) + throw new AssertionError("Unexpected capability manifest: " + response.body()); + if (!response.headers().firstValue("content-type").orElse("").startsWith("application/json") || + !"no-store".equals(response.headers().firstValue("cache-control").orElse(""))) + throw new AssertionError("Capability response headers missing"); + status(400, client.send(signedUri(URI.create(uri + "?extra=1"), "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + } + + private static void status(int expected, HttpResponse response) { if (response.statusCode() != expected) { throw new AssertionError("Expected HTTP " + expected + ", got " + response.statusCode() - + ": " + new String(response.body(), StandardCharsets.UTF_8)); + + ": " + (response.body() instanceof byte[] bytes + ? new String(bytes, StandardCharsets.UTF_8) : response.body())); } } @@ -167,7 +439,7 @@ public final class HttpTest { status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", "/other/source")), HttpResponse.BodyHandlers.ofByteArray())); - status(400, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", + status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", new byte[0], Map.of("x-amz-copy-source", "/objects/source?versionId=1")), HttpResponse.BodyHandlers.ofByteArray())); status(501, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT", @@ -180,12 +452,26 @@ public final class HttpTest { } private static String encodedChecksum(String algorithm, byte[] body) throws Exception { + if (algorithm.startsWith("XXHASH")) { + var checksum = new XxHashes(algorithm); + checksum.update(body, 0, body.length); + return Base64.getEncoder().encodeToString(checksum.digest()); + } if (algorithm.startsWith("CRC")) { - Checksum checksum = algorithm.equals("CRC32") ? new CRC32() : new CRC32C(); + Checksum checksum = switch (algorithm) { + case "CRC32" -> new CRC32(); + case "CRC32C" -> new CRC32C(); + case "CRC64NVME" -> new Crc64Nvme(); + default -> throw new IllegalArgumentException(algorithm); + }; checksum.update(body, 0, body.length); long value = checksum.getValue(); - return Base64.getEncoder().encodeToString(new byte[]{(byte) (value >>> 24), - (byte) (value >>> 16), (byte) (value >>> 8), (byte) value}); + byte[] bytes = new byte[algorithm.equals("CRC64NVME") ? 8 : 4]; + for (int i = bytes.length - 1; i >= 0; i--) { + bytes[i] = (byte) value; + value >>>= 8; + } + return Base64.getEncoder().encodeToString(bytes); } String name = algorithm.equals("SHA1") ? "SHA-1" : algorithm.equals("SHA256") ? "SHA-256" : @@ -197,7 +483,8 @@ public final class HttpTest { URI uri = URI.create(base + "/objects/checksum-target"); byte[] body = "checksum payload".getBytes(StandardCharsets.UTF_8); String md5 = encodedChecksum("MD5", body); - for (String algorithm : new String[]{"CRC32", "CRC32C", "SHA1", "SHA256", "SHA512", "MD5"}) { + for (String algorithm : new String[]{"CRC32", "CRC32C", "CRC64NVME", "XXHASH64", + "XXHASH3", "XXHASH128", "SHA1", "SHA256", "SHA512", "MD5"}) { String header = "x-amz-checksum-" + algorithm.toLowerCase(java.util.Locale.ROOT); String checksum = encodedChecksum(algorithm, body); var stored = client.send(signedUri(uri, "PUT", body, @@ -207,7 +494,9 @@ public final class HttpTest { if (!checksum.equals(stored.headers().firstValue(header).orElse(""))) throw new AssertionError("Missing checksum response: " + algorithm); var bad = client.send(signedUri(uri, "PUT", body, - Map.of(header, Base64.getEncoder().encodeToString(new byte[algorithm.startsWith("CRC") ? 4 : + Map.of(header, Base64.getEncoder().encodeToString(new byte[algorithm.equals("XXHASH128") ? 16 : + algorithm.startsWith("XXHASH") || algorithm.equals("CRC64NVME") ? 8 : + algorithm.startsWith("CRC") ? 4 : algorithm.equals("SHA1") ? 20 : algorithm.equals("SHA256") ? 32 : algorithm.equals("SHA512") ? 64 : 16]))), HttpResponse.BodyHandlers.ofString()); if (bad.statusCode() != 400 || !bad.body().contains("BadDigest")) @@ -217,6 +506,13 @@ public final class HttpTest { status(200, unchanged); if (!java.util.Arrays.equals(body, unchanged.body())) throw new AssertionError("Bad checksum replaced stored object"); + var head = client.send(signedUri(uri, "HEAD", new byte[0], + Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding()); + status(200, head); + if (!checksum.equals(head.headers().firstValue(header).orElse(""))) + throw new AssertionError("Checksum was not persisted: " + algorithm); + status(400, client.send(signedUri(uri, "HEAD", new byte[0], + Map.of("x-amz-checksum-mode", "INVALID")), HttpResponse.BodyHandlers.discarding())); } var badMd5 = client.send(signedUri(uri, "PUT", body, Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString()); @@ -227,8 +523,14 @@ public final class HttpTest { status(400, client.send(signedUri(uri, "PUT", body, Map.of("x-amz-checksum-crc32", encodedChecksum("CRC32", body), "x-amz-sdk-checksum-algorithm", "CRC32C")), HttpResponse.BodyHandlers.ofByteArray())); - status(501, client.send(signedUri(uri, "PUT", body, - Map.of("x-amz-checksum-crc64nvme", "AAAAAAAAAAA=")), HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(uri, "PUT", body, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var automatic = client.send(signedUri(uri, "HEAD", new byte[0], + Map.of("x-amz-checksum-mode", "ENABLED")), HttpResponse.BodyHandlers.discarding()); + status(200, automatic); + if (!encodedChecksum("CRC64NVME", body).equals(automatic.headers() + .firstValue("x-amz-checksum-crc64nvme").orElse(""))) + throw new AssertionError("Default CRC64NVME checksum was not persisted"); status(204, client.send(signedUri(uri, "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); } @@ -296,6 +598,214 @@ public final class HttpTest { HttpResponse.BodyHandlers.ofByteArray())); } + private static void testAttributes(HttpClient client, String base) throws Exception { + URI object = URI.create(base + "/objects/attributes.txt"); + byte[] body = "object attributes".getBytes(StandardCharsets.UTF_8); + status(200, client.send(signedUri(object, "PUT", body, Map.of( + "x-amz-meta-project", "LunarSky", "x-amz-tagging", "kind=test&phase=one")), + HttpResponse.BodyHandlers.ofByteArray())); + var get = client.send(signedUri(object, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, get); + if (!"LunarSky".equals(get.headers().firstValue("x-amz-meta-project").orElse("")) || + !"2".equals(get.headers().firstValue("x-amz-tagging-count").orElse(""))) + throw new AssertionError("Stored attributes missing from GET"); + var tagging = URI.create(object + "?tagging"); + var originalTags = client.send(signedUri(URI.create(tagging + "&x-id=GetObjectTagging"), + "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofString()); + status(200, originalTags); + if (!originalTags.body().contains("kindtest")) + throw new AssertionError("Object tags were not stored"); + byte[] replacement = "stagetwo" + .getBytes(StandardCharsets.UTF_8); + status(200, client.send(signedUri(tagging, "PUT", replacement, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(400, client.send(signedUri(tagging, "PUT", replacement, + Map.of("content-md5", Base64.getEncoder().encodeToString(new byte[16]))), + HttpResponse.BodyHandlers.ofByteArray())); + var replaced = client.send(signedUri(tagging, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofString()); + if (!replaced.body().contains("stagetwo") || + replaced.body().contains("kind")) throw new AssertionError("Tag replacement failed"); + status(400, client.send(signedUri(tagging, "PUT", "]>" + .getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + status(204, client.send(signedUri(tagging, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var cleared = client.send(signedUri(tagging, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofString()); + if (!cleared.body().contains("")) throw new AssertionError("Tag deletion failed"); + status(200, client.send(signedUri(URI.create(base + "/objects/attributes-copy.txt"), "PUT", + new byte[0], Map.of("x-amz-copy-source", "/objects/attributes.txt")), + HttpResponse.BodyHandlers.ofByteArray())); + var copied = client.send(signed(base, "GET", "attributes-copy.txt", new byte[0]), + HttpResponse.BodyHandlers.ofByteArray()); + if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse(""))) + throw new AssertionError("Copy lost user metadata"); + status(400, client.send(signedUri(object, "PUT", body, + Map.of("x-amz-meta-bad", "a".repeat(2100))), HttpResponse.BodyHandlers.ofByteArray())); + } + + private static void testBuckets(HttpClient client, String base) throws Exception { + var root = URI.create(base + "/"); + var existing = client.send(signedUri(root, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofString()); + status(200, existing); + if (!existing.body().contains("objects")) + throw new AssertionError("Default bucket absent from service listing"); + var bucket = URI.create(base + "/second-bucket"); + status(200, client.send(signedUri(URI.create(bucket + "?x-id=CreateBucket"), + "PUT", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(409, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(200, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var upload = client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploads"), + "POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + status(200, upload); + String uploadId = upload.body().split("")[1].split("")[0]; + status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(204, client.send(signedUri(URI.create(base + "/second-bucket/staged.txt?uploadId=" + uploadId), + "DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + byte[] body = "second bucket".getBytes(StandardCharsets.UTF_8); + var object = URI.create(base + "/second-bucket/one.txt"); + status(200, client.send(signedUri(object, "PUT", body, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(409, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var read = client.send(signedUri(object, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, read); + if (!java.util.Arrays.equals(body, read.body())) throw new AssertionError("Second bucket read failed"); + var crossCopy = URI.create(base + "/second-bucket/copied.txt"); + status(200, client.send(signedUri(crossCopy, "PUT", new byte[0], + Map.of("x-amz-copy-source", "/objects/attributes.txt")), + HttpResponse.BodyHandlers.ofByteArray())); + var copied = client.send(signedUri(crossCopy, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + if (!"LunarSky".equals(copied.headers().firstValue("x-amz-meta-project").orElse(""))) + throw new AssertionError("Cross-bucket copy lost metadata"); + var listed = client.send(signedUri(URI.create(base + "/second-bucket?list-type=2"), + "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + if (!listed.body().contains("one.txt")) throw new AssertionError("Second bucket listing failed"); + status(204, client.send(signedUri(object, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(204, client.send(signedUri(crossCopy, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(204, client.send(signedUri(bucket, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + status(404, client.send(signedUri(bucket, "HEAD", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + } + + private static void testVersioning(HttpClient client, String base) throws Exception { + URI bucket = URI.create(base + "/version-bucket"); + URI configuration = URI.create(bucket + "?versioning"); + URI object = URI.create(bucket + "/note.txt"); + status(200, client.send(signedUri(bucket, "PUT", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var initial = client.send(signedUri(configuration, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofString()); + if (!initial.body().contains("")) + throw new AssertionError("New bucket versioning state"); + status(200, client.send(signedUri(object, "PUT", new byte[]{1}, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + byte[] enable = "Enabled" + .getBytes(StandardCharsets.UTF_8); + status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var first = client.send(signedUri(object, "PUT", new byte[]{2}, Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, first); + String id = first.headers().firstValue("x-amz-version-id").orElseThrow(); + var second = client.send(signedUri(object, "PUT", new byte[]{3}, Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, second); + if (id.equals(second.headers().firstValue("x-amz-version-id").orElseThrow())) + throw new AssertionError("Version IDs repeated"); + var old = client.send(signedUri(URI.create(object + "?versionId=" + id), "GET", + new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); + status(200, old); + if (old.body()[0] != 2) throw new AssertionError("Historical object body"); + byte[] tagged = "kindold" + .getBytes(StandardCharsets.UTF_8); + status(200, client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "PUT", + tagged, Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + var oldTags = client.send(signedUri(URI.create(object + "?tagging&versionId=" + id), "GET", + new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + status(200, oldTags); + if (!oldTags.body().contains("old")) + throw new AssertionError("Versioned tagging failed"); + var copied = client.send(signedUri(URI.create(bucket + "/copied.txt"), "PUT", + new byte[0], Map.of("x-amz-copy-source", "/version-bucket/note.txt?versionId=" + id)), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, copied); + if (!id.equals(copied.headers().firstValue("x-amz-copy-source-version-id").orElse(""))) + throw new AssertionError("Copy did not report source version"); + var copyBody = client.send(signedUri(URI.create(bucket + "/copied.txt"), "GET", + new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); + status(200, copyBody); + if (copyBody.body()[0] != 2) throw new AssertionError("Copy of old version failed"); + var deleted = client.send(signedUri(object, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(204, deleted); + String marker = deleted.headers().firstValue("x-amz-version-id").orElseThrow(); + var hidden = client.send(signedUri(object, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(404, hidden); + if (!"true".equals(hidden.headers().firstValue("x-amz-delete-marker").orElse(""))) + throw new AssertionError("Missing delete marker response header"); + status(405, client.send(signedUri(URI.create(object + "?versionId=" + marker), "GET", + new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + var listed = client.send(signedUri(URI.create(bucket + "?versions&max-keys=2"), "GET", + new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + status(200, listed); + if (!listed.body().contains("") || !listed.body().contains("true")) + throw new AssertionError("Version listing did not include delete marker"); + status(204, client.send(signedUri(URI.create(object + "?versionId=" + marker), "DELETE", + new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray())); + var restored = client.send(signedUri(object, "GET", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, restored); + if (restored.body()[0] != 3) throw new AssertionError("Deleting marker did not restore current version"); + byte[] suspend = "Suspended" + .getBytes(StandardCharsets.UTF_8); + status(200, client.send(signedUri(configuration, "PUT", suspend, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var nullVersion = client.send(signedUri(object, "PUT", new byte[]{4}, Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, nullVersion); + if (!"null".equals(nullVersion.headers().firstValue("x-amz-version-id").orElse(""))) + throw new AssertionError("Suspended write did not produce null version"); + status(200, client.send(signedUri(configuration, "PUT", enable, Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + URI multipartObject = URI.create(bucket + "/multipart.txt"); + var initiated = client.send(signedUri(URI.create(multipartObject + "?uploads"), "POST", + new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString()); + status(200, initiated); + String uploadId = initiated.body().split("")[1].split("")[0]; + byte[] partBody = "versioned multipart".getBytes(StandardCharsets.UTF_8); + var part = client.send(signedUri(URI.create(multipartObject + "?partNumber=1&uploadId=" + uploadId), + "PUT", partBody, Map.of()), HttpResponse.BodyHandlers.ofByteArray()); + status(200, part); + String completion = "1" + + part.headers().firstValue("etag").orElseThrow() + ""; + var completed = client.send(signedUri(URI.create(multipartObject + "?uploadId=" + uploadId), + "POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), + HttpResponse.BodyHandlers.ofByteArray()); + status(200, completed); + String multipartVersion = completed.headers().firstValue("x-amz-version-id").orElseThrow(); + status(204, client.send(signedUri(multipartObject, "DELETE", new byte[0], Map.of()), + HttpResponse.BodyHandlers.ofByteArray())); + var retainedMultipart = client.send(signedUri(URI.create(multipartObject + "?versionId=" + + multipartVersion), "GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()); + status(200, retainedMultipart); + if (!java.util.Arrays.equals(partBody, retainedMultipart.body())) + throw new AssertionError("Completed multipart version was not retained"); + } + public static void main(String[] args) throws Exception { Path root = Files.createTempDirectory("store-http-test-"); var executor = Executors.newVirtualThreadPerTaskExecutor(); @@ -303,19 +813,28 @@ public final class HttpTest { DiskStore store = new DiskStore(root, 1024, 4096); try { var app = new Main(store, - new SigV4(ACCESS, SECRET, REGION, Clock.systemUTC()), "objects"); + new SigV4(Map.of(ACCESS, SECRET, SECONDARY, SECONDARY_SECRET), + ACCESS, REGION, Clock.systemUTC()), "objects"); server.setExecutor(executor); server.createContext("/", app::handle); server.start(); String base = "http://127.0.0.1:" + server.getAddress().getPort(); HttpClient client = HttpClient.newHttpClient(); + testCapabilities(client, base); + testPresigned(client, base); + testStreaming(client, base); + testStreamingTrailer(client, base); testObjects(client, base); testListing(client, base); testCopy(client, base); testChecksums(client, base); testMultipart(client, base); + testAttributes(client, base); testDelete(client, base); - System.out.println("HTTP tests passed: objects, copy, checksums, listing, multipart"); + testBuckets(client, base); + testVersioning(client, base); + testAcl(client, base); + System.out.println("HTTP tests passed: capabilities, objects, copy, checksums, listing, multipart, attributes, buckets, versioning, ACLs"); } finally { server.stop(0); executor.close(); diff --git a/test/cloud/lunarsky/store/StoreTest.java b/test/cloud/lunarsky/store/StoreTest.java index 3dff481..ccc7475 100644 --- a/test/cloud/lunarsky/store/StoreTest.java +++ b/test/cloud/lunarsky/store/StoreTest.java @@ -5,6 +5,7 @@ import java.util.Arrays; import java.nio.ByteBuffer; import java.security.MessageDigest; import java.util.List; +import java.util.Map; public final class StoreTest { interface Operation {void run() throws Exception;} @@ -36,9 +37,87 @@ public final class StoreTest { fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers)); headers.add("host","duplicate"); fails(403,()->auth.verify("GET",uri,headers)); + var presignedHeaders = new com.sun.net.httpserver.Headers(); + presignedHeaders.set("host", "examplebucket.s3.amazonaws.com"); + var presigned = java.net.URI.create("/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256" + + "&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request" + + "&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host" + + "&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404"); + if (!"UNSIGNED-PAYLOAD".equals(auth.verifyRequest("GET", presigned, presignedHeaders).payload())) + throw new AssertionError("Official presigned URL was not accepted"); + fails(403, () -> auth.verifyRequest("PUT", presigned, presignedHeaders)); + fails(403, () -> new SigV4("AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1", + java.time.Clock.fixed(java.time.Instant.parse("2013-05-25T00:00:01Z"), + java.time.ZoneOffset.UTC)).verifyRequest("GET", presigned, presignedHeaders)); System.out.println("SigV4 official vector and tampering tests passed"); } + private static void testAwsChunked() throws Exception { + String secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + String date = "20130524T000000Z"; + String scope = "20130524/us-east-1/s3/aws4_request"; + byte[] key = SigV4.signingKey(secret, "20130524", "us-east-1"); + byte[] body = new byte[66560]; + Arrays.fill(body, (byte) 'a'); + var encoded = new ByteArrayOutputStream(); + encoded.write("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n".getBytes()); + encoded.write(body, 0, 65536); + encoded.write("\r\n400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n".getBytes()); + encoded.write(body, 65536, 1024); + encoded.write("\r\n0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n".getBytes()); + var authorization = new SigV4.Verified("STREAMING-AWS4-HMAC-SHA256-PAYLOAD", "", key, + date, scope, "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9"); + try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(encoded.toByteArray()), + authorization, body.length, null)) { + if (!Arrays.equals(body, stream.readAllBytes())) throw new AssertionError("Signed chunk vector mismatch"); + } + byte[] tampered = encoded.toByteArray(); + tampered[100] = 'b'; + fails(400, () -> { + try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(tampered), + authorization, body.length, null)) { stream.readAllBytes(); } + }); + var withTrailer = new ByteArrayOutputStream(); + withTrailer.write("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n".getBytes()); + withTrailer.write(body, 0, 65536); + withTrailer.write("\r\n400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n".getBytes()); + withTrailer.write(body, 65536, 1024); + withTrailer.write("\r\n0;chunk-signature=2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992\r\n".getBytes()); + withTrailer.write("x-amz-checksum-crc32c:sOO8/Q==\r\n".getBytes()); + withTrailer.write("x-amz-trailer-signature=d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435\r\n\r\n".getBytes()); + var trailerAuthorization = new SigV4.Verified("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER", + "", key, date, scope, "106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e"); + try (var stream = new AwsChunkedInputStream(new ByteArrayInputStream(withTrailer.toByteArray()), + trailerAuthorization, body.length, "x-amz-checksum-crc32c")) { + if (!Arrays.equals(body, stream.readAllBytes()) || !"sOO8/Q==".equals(stream.trailerValue())) + throw new AssertionError("Signed checksum trailer vector mismatch"); + } + } + + private static void testCrc64Nvme() { + byte[] zeros = new byte[32]; + Crc64Nvme checksum = new Crc64Nvme(); + checksum.update(zeros, 0, zeros.length); + if (checksum.getValue() != 0xcf3473434d4ecf3bL) + throw new AssertionError("CRC64NVME test vector mismatch"); + } + + private static void testXxHashes() { + byte[] body = "abc".getBytes(java.nio.charset.StandardCharsets.US_ASCII); + Map vectors = Map.of( + "XXHASH64", "44bc2cf5ad770999", + "XXHASH3", "78af5f94892f3950", + "XXHASH128", "06b05ab6733a618578af5f94892f3950"); + for (var entry : vectors.entrySet()) { + XxHashes hash = new XxHashes(entry.getKey()); + hash.update(body, 0, 1); + hash.update(body, 1, 2); + if (!SigV4.hex(hash.digest()).equals(entry.getValue())) + throw new AssertionError(entry.getKey() + " test vector mismatch"); + } + } + private static void testInitialStore(Path root) throws Exception { try(var store=new DiskStore(root,8,10)){ byte[] body={1,2,3,4,5,6}; @@ -134,14 +213,219 @@ public final class StoreTest { try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");} } + private static void testAttributesRestart(Path root) throws Exception { + Path data = root.resolve("attributes"); + byte[] body = {1, 2, 3}; + var crc = new Crc64Nvme(); + crc.update(body, 0, body.length); + byte[] digest = ByteBuffer.allocate(8).putLong(crc.getValue()).array(); + Map checksums = Map.of("x-amz-checksum-crc64nvme", + java.util.Base64.getEncoder().encodeToString(digest)); + try (var store = new DiskStore(data, 8, 10)) { + store.put("test", "metadata", new ByteArrayInputStream(body), body.length, + SigV4.hex(SigV4.hash(body)), null, false, "text/plain", + Map.of("project", "LunarSky"), Map.of("stage", "one"), () -> checksums); + store.setTags("test", "metadata", Map.of("stage", "two")); + } + try (var store = new DiskStore(data, 8, 10); + var object = store.open("test", "metadata")) { + if (!Arrays.equals(body, object.stream().readAllBytes()) || + !object.metadata().userMetadata().equals(Map.of("project", "LunarSky")) || + !object.metadata().tags().equals(Map.of("stage", "two")) || + !object.metadata().checksums().equals(checksums)) + throw new AssertionError("Object attributes were lost after restart"); + } + } + + private static void testV3Record(Path root) throws Exception { + Path data = root.resolve("v3-record"); + String bucket = "test", key = "v3-object"; + byte[] body = {8, 9, 10}; + byte[] bucketBytes = bucket.getBytes(java.nio.charset.StandardCharsets.UTF_8); + byte[] keyBytes = key.getBytes(java.nio.charset.StandardCharsets.UTF_8); + byte[] typeBytes = "text/plain".getBytes(java.nio.charset.StandardCharsets.UTF_8); + byte[] metadata = ObjectAttributes.encode(Map.of("legacy", "yes"), 4096); + byte[] tags = ObjectAttributes.encode(Map.of("source", "v3"), 8192); + String pathHash = SigV4.hex(SigV4.hash((bucket + "/" + key).getBytes(java.nio.charset.StandardCharsets.UTF_8))); + Path path = data.resolve("objects").resolve(pathHash.substring(0, 2)).resolve(pathHash); + Files.createDirectories(path.getParent()); + ByteBuffer bytes = ByteBuffer.allocate(82 + bucketBytes.length + keyBytes.length + + typeBytes.length + metadata.length + tags.length + body.length); + bytes.putLong(0x4c534f424a303033L).putLong(body.length).putLong(123456789L) + .put(MessageDigest.getInstance("MD5").digest(body)).put(SigV4.hash(body)) + .putShort((short) bucketBytes.length).putShort((short) keyBytes.length) + .putShort((short) typeBytes.length).putShort((short) metadata.length) + .putShort((short) tags.length).put(bucketBytes).put(keyBytes).put(typeBytes) + .put(metadata).put(tags).put(body); + Files.write(path, bytes.array()); + try (var store = new DiskStore(data, 64, 128)) { + try (var object = store.open(bucket, key)) { + if (!Arrays.equals(body, object.stream().readAllBytes()) || + !object.metadata().userMetadata().equals(Map.of("legacy", "yes")) || + !object.metadata().tags().equals(Map.of("source", "v3"))) + throw new AssertionError("V3 record was not readable"); + } + store.setTags(bucket, key, Map.of("source", "v4")); + } + try (var store = new DiskStore(data, 64, 128); + var object = store.open(bucket, key)) { + if (!Arrays.equals(body, object.stream().readAllBytes()) || + !object.metadata().tags().equals(Map.of("source", "v4"))) + throw new AssertionError("V3 record upgrade failed"); + } + } + + private static void testBucketRestart(Path root) throws Exception { + Path data = root.resolve("buckets"); + byte[] body = {4, 5, 6}; + try (var store = new DiskStore(data, 8, 10)) { + store.ensureBucket("default-bucket"); + store.createBucket("second-bucket"); + store.put("second-bucket", "object", new ByteArrayInputStream(body), body.length, + SigV4.hex(SigV4.hash(body)), null, false, "application/octet-stream"); + } + try (var store = new DiskStore(data, 8, 10); + var object = store.open("second-bucket", "object")) { + if (store.buckets().size() != 2 || !Arrays.equals(body, object.stream().readAllBytes())) + throw new AssertionError("Bucket catalog was lost after restart"); + fails(409, () -> store.deleteBucket("second-bucket")); + store.delete("second-bucket", "object"); + store.deleteBucket("second-bucket"); + fails(404, () -> store.bucket("second-bucket")); + } + } + + private static void testAclPersistence(Path root) throws Exception { + Path data = root.resolve("acl"); + Map grant = Map.of("SECONDARYKEY1234", Integer.toString(Acl.READ)); + byte[] body = {9, 8, 7}; + String older; + try (var store = new DiskStore(data, 32, 128)) { + store.ensureBucket("acl-bucket"); + store.setBucketAcl("acl-bucket", grant); + store.setVersioning("acl-bucket", ObjectStorage.VersioningState.ENABLED); + older = store.put("acl-bucket", "image", new ByteArrayInputStream(body), body.length, + SigV4.hex(SigV4.hash(body)), null, false, "image/png", + Map.of(), Map.of(), Map::of, grant).versionId(); + store.put("acl-bucket", "image", new ByteArrayInputStream(body), body.length, + SigV4.hex(SigV4.hash(body)), null, false, "image/png"); + } + try (var store = new DiskStore(data, 32, 128); + var previous = store.open("acl-bucket", "image", older); + var current = store.open("acl-bucket", "image")) { + if (!store.bucket("acl-bucket").acl().equals(grant) || + !previous.metadata().acl().equals(grant) || !current.metadata().acl().isEmpty()) + throw new AssertionError("ACL grants changed after restart or version replacement"); + store.setObjectAcl("acl-bucket", "image", older, Map.of(Acl.ALL_USERS, "1")); + } + try (var store = new DiskStore(data, 32, 128); + var previous = store.open("acl-bucket", "image", older)) { + if (!previous.metadata().acl().equals(Map.of(Acl.ALL_USERS, "1"))) + throw new AssertionError("Version-specific ACL edit was not durable"); + } + } + + private static void testAdditionalKeys(Path root) throws Exception { + Path file = root.resolve("access-keys"); + Files.writeString(file, "SECONDARYKEY1234:secondary-secret-key-that-is-at-least-32-characters\n"); + Map keys = Main.identities( + Map.of("S3_CREDENTIALS_FILE", file.toString()), + "TESTACCESSKEY123", "test-secret-key-that-is-at-least-32-characters"); + if (keys.size() != 2 || !keys.containsKey("SECONDARYKEY1234")) + throw new AssertionError("Additional access key was not loaded"); + Files.writeString(file, "TESTACCESSKEY123:duplicate-root-secret-that-is-at-least-32-characters\n"); + try { + Main.identities(Map.of("S3_CREDENTIALS_FILE", file.toString()), + "TESTACCESSKEY123", "test-secret-key-that-is-at-least-32-characters"); + throw new AssertionError("Duplicate root key was accepted"); + } catch (IllegalArgumentException expected) { } + } + + private static void testVersioning(Path root) throws Exception { + Path data = root.resolve("versioning"); + String first; + String second; + String marker; + try (var store = new DiskStore(data, 8, 100)) { + store.createBucket("versioned-bucket"); + byte[] old = {1}; + store.put("versioned-bucket", "note", new ByteArrayInputStream(old), old.length, + SigV4.hex(SigV4.hash(old)), null, false, "text/plain"); + store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.ENABLED); + byte[] newer = {2}; + first = store.put("versioned-bucket", "note", new ByteArrayInputStream(newer), newer.length, + SigV4.hex(SigV4.hash(newer)), null, false, "text/plain").versionId(); + byte[] latest = {3}; + second = store.put("versioned-bucket", "note", new ByteArrayInputStream(latest), latest.length, + SigV4.hex(SigV4.hash(latest)), null, false, "text/plain").versionId(); + if (first == null || second == null || first.equals(second)) throw new AssertionError("Unique versions"); + if (store.usedBytes() != 3) throw new AssertionError("Retained versions did not count toward capacity"); + try (var object = store.open("versioned-bucket", "note", first)) { + if (object.stream().read() != 2) throw new AssertionError("Old version was overwritten"); + } + marker = store.delete("versioned-bucket", "note", null).versionId(); + fails(404, () -> store.open("versioned-bucket", "note")); + if (!store.list("versioned-bucket", "", "", 10, null).objects().isEmpty()) + throw new AssertionError("Delete marker appeared in current listing"); + var page = store.listVersions("versioned-bucket", "", null, null, 2); + if (!page.truncated() || !page.entries().getFirst().deleteMarker() || + !page.entries().get(1).versionId().equals(second)) throw new AssertionError("Version listing"); + var rest = store.listVersions("versioned-bucket", "", page.nextKey(), page.nextVersionId(), 10); + if (rest.entries().size() != 2 || !rest.entries().getLast().versionId().equals("null")) + throw new AssertionError("Version pagination"); + store.delete("versioned-bucket", "note", marker); + try (var object = store.open("versioned-bucket", "note")) { + if (object.stream().read() != 3) throw new AssertionError("Delete marker removal"); + } + store.setVersioning("versioned-bucket", ObjectStorage.VersioningState.SUSPENDED); + byte[] suspended = {4}; + var nullVersion = store.put("versioned-bucket", "note", new ByteArrayInputStream(suspended), + suspended.length, SigV4.hex(SigV4.hash(suspended)), null, false, "text/plain"); + if (!"null".equals(nullVersion.versionId())) throw new AssertionError("Suspended null version"); + if (store.usedBytes() != 3) throw new AssertionError("Suspended write did not replace null version"); + try (var object = store.open("versioned-bucket", "note", second)) { + if (object.stream().read() != 3) throw new AssertionError("Suspension removed a version"); + } + store.setTags("versioned-bucket", "note", Map.of("stage", "suspended")); + } + try (var store = new DiskStore(data, 8, 100)) { + if (store.bucket("versioned-bucket").versioning() != ObjectStorage.VersioningState.SUSPENDED) + throw new AssertionError("Versioning state was lost"); + try (var object = store.open("versioned-bucket", "note")) { + if (object.stream().read() != 4 || !object.metadata().tags().equals(Map.of("stage", "suspended"))) + throw new AssertionError("Versioned object was lost"); + } + store.delete("versioned-bucket", "note"); + fails(404, () -> store.open("versioned-bucket", "note")); + if (store.usedBytes() != 2) throw new AssertionError("Suspended delete did not release null version"); + try (var object = store.open("versioned-bucket", "note", second)) { + if (object.stream().read() != 3) throw new AssertionError("Suspended delete removed old version"); + } + fails(409, () -> store.deleteBucket("versioned-bucket")); + for (var entry : store.listVersions("versioned-bucket", "", null, null, 10).entries()) + store.delete("versioned-bucket", "note", entry.versionId()); + if (store.usedBytes() != 0) throw new AssertionError("Version deletes did not release capacity"); + store.deleteBucket("versioned-bucket"); + } + } + public static void main(String[] args) throws Exception { testSignature(); + testAwsChunked(); + testCrc64Nvme(); + testXxHashes(); Path root = Files.createTempDirectory("store-test-"); try { testInitialStore(root); testRestart(root); testMultipartRecovery(root); testLegacyRecord(root); + testAttributesRestart(root); + testV3Record(root); + testBucketRestart(root); + testAclPersistence(root); + testAdditionalKeys(root); + testVersioning(root); testCorruption(root); System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete"); } finally { diff --git a/tests/two-host/README.md b/tests/two-host/README.md new file mode 100644 index 0000000..78fa9f3 --- /dev/null +++ b/tests/two-host/README.md @@ -0,0 +1,50 @@ +# Two-machine durability drill + +Run this disposable test on two machines. Machine A runs the gateway, PostgreSQL, and one storage node; machine B runs a second storage node. Keep the node connection on a private network: the node protocol uses bearer tokens over HTTP. + +Both machines need Docker. Machine A also needs Docker Compose and Python 3. The example uses loopback port 9003 on machine A and private-network port 9103 on machine B; change them if needed. Use separate test volumes, and do not point this drill at an existing ObjectStore cluster. + +## Start the cluster + +On machine A, generate test-only credentials outside the repository and build the current image: + +```sh +python3 tests/two-host/prepare.py /path/to/private-test-dir http://MACHINE_B_PRIVATE_IP:9103 +docker build -t objectstore-durability:local . +docker save objectstore-durability:local | gzip > /path/to/private-test-dir/objectstore-durability.tar.gz +``` + +Transfer the image archive and `remote-node.env` to a private directory on machine B. Load the image there, then start its node with a dedicated volume and a port bound only to its private-network address: + +```sh +gzip -dc objectstore-durability.tar.gz | sudo docker load +sudo docker run -d --name objectstore-durability-remote --restart unless-stopped \ + --publish MACHINE_B_PRIVATE_IP:9103:9100 \ + --volume objectstore-durability-remote-data:/data \ + --env-file remote-node.env --entrypoint java objectstore-durability:local \ + --add-modules jdk.httpserver,java.net.http -cp /app:/app/postgresql.jar \ + cloud.lunarsky.store.ClusterNode +``` + +Start the services on machine A from the repository root: + +```sh +docker compose --env-file /path/to/private-test-dir/cluster.env \ + -f tests/two-host/compose.yaml up -d --wait gateway +python3 tests/two-host/check.py /path/to/private-test-dir/cluster.env \ + /path/to/private-test-dir/acknowledged.jsonl seed +python3 tests/two-host/check.py /path/to/private-test-dir/cluster.env \ + /path/to/private-test-dir/acknowledged.jsonl verify +``` + +Before disrupting either machine, check `cluster_segments.replica_ids` against `cluster_nodes.host_id` in the test PostgreSQL database. Every seeded segment must have replicas on two distinct host IDs. The IDs are operator labels; confirm that the nodes run on separate machines. + +## Failure checks + +Run `check.py ... stress --seconds 120` while both nodes are healthy, then interrupt machine B. The journal records a write only after an HTTP 200 response and calls `fsync` for each entry. New writes should return 503 while only one machine remains. Reads of acknowledged objects should still succeed from the other replica. Restore machine B, wait for its node to answer `/health`, and run `check.py ... verify` against the entire journal. + +Stop the node container on machine A and repeat the rejection and read checks using the replica on machine B. Restart the node, then abruptly kill only the disposable gateway, metadata, and node containers on machine A during another stress run. Start them again and verify the journal. A connection closed during a write has an **uncertain** outcome; do not count it as acknowledged or assume it was rejected. + +For a manual metadata recovery drill, take a custom-format `pg_dump` after the last acknowledged write and verify it with `pg_restore --list`. Copy the dump to machine B and restore it into a fresh PostgreSQL volume there. Stop the original gateway and metadata container on machine A. Start a separate gateway against the restored database, then verify the journal. Finally stop the storage node on machine A and verify again: this forces reads to use both the restored metadata and the replica on machine B. Keep the backup and its credentials private. + +Passing this drill does not prove every crash point, disk-controller write behavior, long-term bit-rot resistance, automatic metadata failover, or production readiness. The cluster still requires a manual metadata restore. diff --git a/tests/two-host/check.py b/tests/two-host/check.py new file mode 100644 index 0000000..7f5b0fb --- /dev/null +++ b/tests/two-host/check.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +import argparse +import datetime +import hashlib +import hmac +import http.client +import json +import os +from pathlib import Path +import time +import urllib.parse + + +def load_env(path): + return dict(line.split("=", 1) for line in Path(path).read_text().splitlines() + if line and not line.startswith("#")) + + +def sign(key, value): + return hmac.new(key, value.encode(), hashlib.sha256).digest() + + +def request(env, method, key, body=b""): + port = int(env.get("CLUSTER_HOST_PORT", "9003")) + host = f"127.0.0.1:{port}" + path = "/durability/" + urllib.parse.quote(key, safe="/-_.~") + date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + stamp = date[:8] + digest = hashlib.sha256(body).hexdigest() + headers = {"host": host, "x-amz-content-sha256": digest, "x-amz-date": date} + signed = ";".join(sorted(headers)) + canonical_headers = "".join(f"{name}:{headers[name]}\n" for name in sorted(headers)) + canonical = f"{method}\n{path}\n\n{canonical_headers}\n{signed}\n{digest}" + scope = f"{stamp}/us-east-1/s3/aws4_request" + to_sign = f"AWS4-HMAC-SHA256\n{date}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}" + key_bytes = ("AWS4" + env["S3_SECRET_KEY"]).encode() + for component in (stamp, "us-east-1", "s3", "aws4_request"): + key_bytes = sign(key_bytes, component) + signature = hmac.new(key_bytes, to_sign.encode(), hashlib.sha256).hexdigest() + headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={env['S3_ACCESS_KEY']}/{scope}," + f"SignedHeaders={signed},Signature={signature}") + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=40) + try: + connection.request(method, path, body=body if method == "PUT" else None, headers=headers) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + +def payload(key, length): + return hashlib.shake_256(key.encode()).digest(length) + + +def record(journal, key, body): + entry = {"key": key, "length": len(body), "sha256": hashlib.sha256(body).hexdigest()} + with open(journal, "a", encoding="utf-8") as output: + output.write(json.dumps(entry, separators=(",", ":")) + "\n") + output.flush() + os.fsync(output.fileno()) + + +def put_and_record(env, journal, key, length): + body = payload(key, length) + try: + status, response = request(env, "PUT", key, body) + except (OSError, TimeoutError) as error: + return "uncertain", str(error) + if status == 200: + record(journal, key, body) + return "acknowledged", "" + return "rejected", f"HTTP {status}: {response[:120]!r}" + + +def seed(env, journal): + for key, length in (("durability/seed-small", 4096), + ("durability/seed-multisegment", 9 * 1024 * 1024 + 17)): + outcome, detail = put_and_record(env, journal, key, length) + if outcome != "acknowledged": + raise RuntimeError(f"Seed {key}: {outcome} {detail}") + print(f"Acknowledged {key}: {length} bytes", flush=True) + + +def stress(env, journal, seconds): + end = time.monotonic() + seconds + counts = {"acknowledged": 0, "rejected": 0, "uncertain": 0} + sequence = 0 + nonce = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S") + while time.monotonic() < end and sequence < 200: + key = f"durability/stress/{nonce}-{sequence:04d}" + outcome, detail = put_and_record(env, journal, key, 256 * 1024) + counts[outcome] += 1 + if outcome != "acknowledged" and counts[outcome] == 1: + print(f"{key}: {outcome} {detail}", flush=True) + sequence += 1 + time.sleep(0.1) + print(json.dumps(counts, sort_keys=True), flush=True) + + +def verify(env, journal): + entries = [json.loads(line) for line in Path(journal).read_text().splitlines() if line] + if not entries: + raise RuntimeError("Journal contains no acknowledged writes") + failures = [] + for entry in entries: + try: + status, body = request(env, "GET", entry["key"]) + actual = hashlib.sha256(body).hexdigest() + if status != 200 or len(body) != entry["length"] or actual != entry["sha256"]: + failures.append(f"{entry['key']}: HTTP {status}, {len(body)} bytes, SHA-256 {actual}") + except (OSError, TimeoutError) as error: + failures.append(f"{entry['key']}: {error}") + for failure in failures: + print(failure) + print(f"Verified {len(entries) - len(failures)}/{len(entries)} acknowledged writes", flush=True) + if failures: + raise SystemExit(1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("env_file") + parser.add_argument("journal") + parser.add_argument("action", choices=("seed", "stress", "verify")) + parser.add_argument("--seconds", type=int, default=90) + args = parser.parse_args() + env = load_env(args.env_file) + if args.action == "seed": + seed(env, args.journal) + elif args.action == "stress": + stress(env, args.journal, args.seconds) + else: + verify(env, args.journal) + + +if __name__ == "__main__": + main() diff --git a/tests/two-host/compose.yaml b/tests/two-host/compose.yaml new file mode 100644 index 0000000..eb97848 --- /dev/null +++ b/tests/two-host/compose.yaml @@ -0,0 +1,120 @@ +services: + metadata: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_DB: objectstore + POSTGRES_USER: objectstore + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + volumes: + - metadata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U objectstore -d objectstore"] + interval: 5s + timeout: 3s + retries: 10 + mem_limit: 256m + security_opt: + - no-new-privileges:true + + node-local: + image: objectstore-durability:local + restart: unless-stopped + entrypoint: ["java", "--add-modules", "jdk.httpserver,java.net.http", "-cp", "/app:/app/postgresql.jar", "cloud.lunarsky.store.ClusterNode"] + environment: + CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN} + CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN} + CLUSTER_HOST_ID: ${LOCAL_HOST_ID:?Set LOCAL_HOST_ID} + NODE_BIND: 0.0.0.0 + DATA_DIR: /data + volumes: + - node-local:/data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/health"] + interval: 5s + timeout: 3s + retries: 10 + mem_limit: 256m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + + gateway: + image: objectstore-durability:local + restart: unless-stopped + environment: + STORE_MODE: cluster + CLUSTER_LOCAL_DEV: "true" + BIND_ADDRESS: 0.0.0.0 + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + S3_BUCKET: durability + S3_REGION: us-east-1 + MAX_OBJECT_BYTES: 67108864 + MAX_TOTAL_BYTES: 536870912 + CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN} + CLUSTER_NODES: http://node-local:9100,${REMOTE_NODE_URL:?Set REMOTE_NODE_URL} + POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10 + POSTGRES_USER: objectstore + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + ports: + - "127.0.0.1:${CLUSTER_HOST_PORT:-9003}:9000" + depends_on: + metadata: + condition: service_healthy + node-local: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9000/ready"] + interval: 5s + timeout: 3s + retries: 10 + mem_limit: 384m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + + repair: + image: objectstore-durability:local + profiles: [repair] + entrypoint: ["/usr/local/bin/objectstore", "cluster-repair"] + environment: + STORE_MODE: cluster + CLUSTER_LOCAL_DEV: "true" + CLUSTER_NODES: http://node-local:9100,${REMOTE_NODE_URL:?Set REMOTE_NODE_URL} + CLUSTER_TOKEN: ${CLUSTER_TOKEN:?Set CLUSTER_TOKEN} + CLUSTER_REPAIR_TOKEN: ${CLUSTER_REPAIR_TOKEN:?Set CLUSTER_REPAIR_TOKEN} + S3_BUCKET: durability + POSTGRES_JDBC_URL: jdbc:postgresql://metadata:5432/objectstore?connectTimeout=3&socketTimeout=10 + POSTGRES_USER: objectstore + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + mem_limit: 384m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + + metadata-recovery: + image: postgres:17-alpine + profiles: [recovery] + environment: + POSTGRES_DB: objectstore + POSTGRES_USER: objectstore + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + volumes: + - metadata-recovery:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U objectstore -d objectstore"] + interval: 5s + timeout: 3s + retries: 10 + mem_limit: 256m + security_opt: + - no-new-privileges:true + +volumes: + metadata: + metadata-recovery: + node-local: diff --git a/tests/two-host/prepare.py b/tests/two-host/prepare.py new file mode 100644 index 0000000..b43d02c --- /dev/null +++ b/tests/two-host/prepare.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import argparse +import os +from pathlib import Path +import secrets +import urllib.parse +import uuid + + +def write_private(path, lines): + with path.open("x", encoding="utf-8") as output: + output.write("\n".join(lines) + "\n") + path.chmod(0o600) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("directory", type=Path) + parser.add_argument("remote_node_url") + parser.add_argument("--gateway-port", type=int, default=9003) + args = parser.parse_args() + url = urllib.parse.urlparse(args.remote_node_url) + if url.scheme != "http" or not url.hostname or not url.port or url.path or url.query or url.fragment: + parser.error("remote_node_url must be an http://host:port address without a path") + if not 1 <= args.gateway_port <= 65535: + parser.error("gateway port must be between 1 and 65535") + args.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + args.directory.chmod(0o700) + access = "DurabilityTest" + secrets.token_hex(8) + secret = secrets.token_hex(32) + token = secrets.token_hex(32) + repair = secrets.token_hex(32) + remote_id = uuid.uuid4() + old_umask = os.umask(0o077) + try: + write_private(args.directory / "cluster.env", [ + "COMPOSE_PROJECT_NAME=objectstore-durability", + f"S3_ACCESS_KEY={access}", f"S3_SECRET_KEY={secret}", + f"CLUSTER_TOKEN={token}", f"CLUSTER_REPAIR_TOKEN={repair}", + f"POSTGRES_PASSWORD={secrets.token_hex(24)}", + f"LOCAL_HOST_ID={uuid.uuid4()}", f"REMOTE_HOST_ID={remote_id}", + f"REMOTE_NODE_URL={args.remote_node_url}", + f"CLUSTER_HOST_PORT={args.gateway_port}", "S3_BUCKET=durability", + ]) + write_private(args.directory / "remote-node.env", [ + f"CLUSTER_TOKEN={token}", f"CLUSTER_REPAIR_TOKEN={repair}", + f"CLUSTER_HOST_ID={remote_id}", "NODE_BIND=0.0.0.0", + "NODE_PORT=9100", "DATA_DIR=/data", + ]) + finally: + os.umask(old_umask) + print(f"Test configuration written to {args.directory}") + + +if __name__ == "__main__": + main()