Address ObjectStore quality findings

2026-10-10 16:07:13 +02:00
1 parent d6427fbac9
commit 885239be1c
@@ -105,42 +105,51 @@ final class AwsChunkedInputStream extends FilterInputStream {
        catch (NumberFormatException error) { throw invalid("Invalid signed chunk size"); }
        if (chunkLeft > decodedLength - decoded) throw invalid("Signed chunks exceed decoded length");
        chunkHash.reset();
-        if (chunkLeft == 0) {
-            finishChunk();
-            if (decoded != decodedLength) throw invalid("Decoded length mismatch");
-            if (trailerName == null) {
-                if (!line().isEmpty()) throw invalid("Invalid signed chunk ending");
-            } else {
-                String trailer = line();
-                if (!trailer.startsWith(trailerName + ":")) throw invalid("Missing signed checksum trailer");
-                trailerValue = trailer.substring(trailerName.length() + 1);
-                byte[] actual;
-                if (trailerCrc != null) {
-                    long value = trailerCrc.getValue();
-                    actual = new byte[trailerName.equals("x-amz-checksum-crc64nvme") ? 8 : 4];
-                    for (int i = actual.length - 1; i >= 0; i--) {
-                        actual[i] = (byte) value;
-                        value >>>= 8;
-                    }
-                } else actual = trailerXxhash != null ? trailerXxhash.digest() : trailerHash.digest();
-                if (!Base64.getEncoder().encodeToString(actual).equals(trailerValue))
-                    throw new StoreException(400, "BadDigest", "Checksum trailer mismatch");
-                String signature = line();
-                if (!signature.matches("x-amz-trailer-signature=[0-9a-f]{64}"))
-                    throw invalid("Missing trailer signature");
-                String toSign = "AWS4-HMAC-SHA256-TRAILER\n" + authorization.date() + "\n" +
-                    authorization.scope() + "\n" + previousSignature + "\n" +
-                    SigV4.hex(SigV4.hash((trailerName + ":" + trailerValue + "\n")
-                        .getBytes(StandardCharsets.UTF_8)));
-                String expected = SigV4.hex(SigV4.hmac(authorization.signingKey(), toSign));
-                if (!MessageDigest.isEqual(expected.getBytes(StandardCharsets.US_ASCII),
-                    signature.substring(24).getBytes(StandardCharsets.US_ASCII)))
-                    throw invalid("Trailer signature mismatch");
-                if (!line().isEmpty()) throw invalid("Invalid trailer ending");
-            }
-            if (in.read() != -1) throw invalid("Extra bytes after signed payload");
-            finished = true;
+        if (chunkLeft == 0) finishPayload();
+    }
+
+    private void finishPayload() throws IOException {
+        finishChunk();
+        if (decoded != decodedLength) throw invalid("Decoded length mismatch");
+        if (trailerName == null) {
+            if (!line().isEmpty()) throw invalid("Invalid signed chunk ending");
+        } else {
+            verifyTrailer();
        }
+        if (in.read() != -1) throw invalid("Extra bytes after signed payload");
+        finished = true;
+    }
+
+    private void verifyTrailer() throws IOException {
+        String trailer = line();
+        if (!trailer.startsWith(trailerName + ":")) throw invalid("Missing signed checksum trailer");
+        trailerValue = trailer.substring(trailerName.length() + 1);
+        if (!Base64.getEncoder().encodeToString(trailerChecksum()).equals(trailerValue))
+            throw new StoreException(400, "BadDigest", "Checksum trailer mismatch");
+        String signature = line();
+        if (!signature.matches("x-amz-trailer-signature=[0-9a-f]{64}"))
+            throw invalid("Missing trailer signature");
+        String toSign = "AWS4-HMAC-SHA256-TRAILER\n" + authorization.date() + "\n" +
+            authorization.scope() + "\n" + previousSignature + "\n" +
+            SigV4.hex(SigV4.hash((trailerName + ":" + trailerValue + "\n")
+                .getBytes(StandardCharsets.UTF_8)));
+        String expected = SigV4.hex(SigV4.hmac(authorization.signingKey(), toSign));
+        if (!MessageDigest.isEqual(expected.getBytes(StandardCharsets.US_ASCII),
+            signature.substring(24).getBytes(StandardCharsets.US_ASCII)))
+            throw invalid("Trailer signature mismatch");
+        if (!line().isEmpty()) throw invalid("Invalid trailer ending");
+    }
+
+    private byte[] trailerChecksum() {
+        if (trailerCrc == null)
+            return trailerXxhash != null ? trailerXxhash.digest() : trailerHash.digest();
+        long value = trailerCrc.getValue();
+        byte[] actual = new byte[trailerName.equals("x-amz-checksum-crc64nvme") ? 8 : 4];
+        for (int i = actual.length - 1; i >= 0; i--) {
+            actual[i] = (byte) value;
+            value >>>= 8;
+        }
+        return actual;
    }

    private void finishChunk() throws IOException {
@@ -114,29 +114,7 @@ final class ClientLimits {
        if (exchange.getRemoteAddress().getAddress().isLoopbackAddress() &&
            exchange.getRequestHeaders().get("X-Real-IP") == null &&
            path.equals("/health")) return null;
-        String address = address(exchange);
-        Client client;
-        synchronized (this) {
-            long now = System.nanoTime();
-            if (++admissions % 1024 == 0 || clients.size() >= MAX_CLIENTS)
-                clients.entrySet().removeIf(entry -> entry.getValue().inFlight == 0 &&
-                    now - entry.getValue().lastSeen > IDLE_NANOS);
-            client = clients.get(address);
-            if (client == null) {
-                if (clients.size() >= MAX_CLIENTS)
-                    throw new StoreException(503, "SlowDown", "Client limit table is full");
-                client = new Client(now, requestBurst, byteBurst);
-                clients.put(address, client);
-            }
-            refill(client, now);
-            client.lastSeen = now;
-            if (maxInFlight > 0 && client.inFlight >= maxInFlight)
-                throw new StoreException(503, "SlowDown", "Too many concurrent requests from this client");
-            if (requestsPerSecond > 0 && client.requestTokens < 1)
-                throw new StoreException(503, "SlowDown", "Client request rate exceeded");
-            if (requestsPerSecond > 0) client.requestTokens--;
-            client.inFlight++;
-        }
+        Client client = admit(address(exchange));
        if (bytesPerSecond > 0) {
            try {
                exchange.setStreams(new LimitedInput(exchange.getRequestBody(), client),
@@ -149,6 +127,30 @@ final class ClientLimits {
        return client;
    }

+    private synchronized Client admit(String address) {
+        Client client;
+        long now = System.nanoTime();
+        if (++admissions % 1024 == 0 || clients.size() >= MAX_CLIENTS)
+            clients.entrySet().removeIf(entry -> entry.getValue().inFlight == 0 &&
+                now - entry.getValue().lastSeen > IDLE_NANOS);
+        client = clients.get(address);
+        if (client == null) {
+            if (clients.size() >= MAX_CLIENTS)
+                throw new StoreException(503, "SlowDown", "Client limit table is full");
+            client = new Client(now, requestBurst, byteBurst);
+            clients.put(address, client);
+        }
+        refill(client, now);
+        client.lastSeen = now;
+        if (maxInFlight > 0 && client.inFlight >= maxInFlight)
+            throw new StoreException(503, "SlowDown", "Too many concurrent requests from this client");
+        if (requestsPerSecond > 0 && client.requestTokens < 1)
+            throw new StoreException(503, "SlowDown", "Client request rate exceeded");
+        if (requestsPerSecond > 0) client.requestTokens--;
+        client.inFlight++;
+        return client;
+    }
+
    synchronized void leave(Client client) {
        if (client != null) {
            client.inFlight--;
@@ -63,55 +63,10 @@ public final class Main {
            admitted = slots.tryAcquire();
            if (!admitted) throw new StoreException(503, "SlowDown", "Too many concurrent requests");
            if (handleStatus(exchange)) return;
-            SigV4.Verified verified = anonymousRead(exchange)
-                ? new SigV4.Verified("UNSIGNED-PAYLOAD", exchange.getRequestURI().getRawQuery(),
-                    null, null, null, null, null)
-                : authentication.verifyRequest(exchange.getRequestMethod(),
-                    exchange.getRequestURI(), exchange.getRequestHeaders());
-            String hash = verified.payload();
-            String principal = verified.principal();
-            String path = SigV4.decode(exchange.getRequestURI().getRawPath());
-            Map<String, String> query = query(verified.applicationQuery());
-            if (path.equals(CAPABILITIES_PATH)) {
-                requireOwner(principal);
-                if (!exchange.getRequestMethod().equals("GET")) unsupported("Capability operation");
-                if (!query.isEmpty())
-                    throw new StoreException(400, "InvalidArgument", "Capability request has unsupported query parameters");
-                requireEmptyBody(exchange, hash);
-                capabilities(exchange);
-            } else if (path.equals("/")) {
-                requireOwner(principal);
-                if (!exchange.getRequestMethod().equals("GET") ||
-                    !(query.isEmpty() || query.size() == 1 && "ListBuckets".equals(query.get("x-id"))))
-                    unsupported("Service operation");
-                requireEmptyBody(exchange, hash);
-                listBuckets(exchange);
-            } else {
-                int slash = path.indexOf('/', 1);
-                String requestedBucket = slash < 0 ? path.substring(1) : path.substring(1, slash);
-                if (requestedBucket.isEmpty()) throw new StoreException(404, "NoSuchBucket", "Bucket not found");
-                if (slash < 0 || slash == path.length() - 1) {
-                    handleBucket(exchange, query, hash, requestedBucket, principal);
-                } else {
-                    store.bucket(requestedBucket);
-                    handleObject(exchange, path.substring(slash + 1), query, verified, requestedBucket);
-                }
-            }
+            dispatch(exchange);
        } catch (StoreException error) {
-            if (error.status == 503 && error.code.equals("SlowDown"))
-                exchange.getResponseHeaders().set("Retry-After", "1");
-            if (anonymousRead(exchange) && error.status == 404)
-                error = new StoreException(403, "AccessDenied", "Access denied");
-            if (error.deleteMarker) {
-                exchange.getResponseHeaders().set("x-amz-delete-marker", "true");
-                exchange.getResponseHeaders().set("x-amz-version-id", error.versionId);
-                if (error.modified >= 0) exchange.getResponseHeaders().set("Last-Modified",
-                    DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC)
-                        .format(Instant.ofEpochMilli(error.modified)));
-            }
-            sendError(exchange, error.status, error.code, error.getMessage(), requestId);
-        }
-        catch (Exception error) {
+            sendStoreError(exchange, error, requestId);
+        } catch (Exception error) {
            System.err.println("ObjectStore request failed: " + requestId + " " + error.getClass().getSimpleName());
            sendError(exchange, 500, "InternalError", "Storage operation failed", requestId);
        } finally {
@@ -123,6 +78,58 @@ public final class Main {
        }
    }

+    private void dispatch(HttpExchange exchange) throws IOException {
+        SigV4.Verified verified = anonymousRead(exchange)
+            ? new SigV4.Verified("UNSIGNED-PAYLOAD", exchange.getRequestURI().getRawQuery(),
+                null, null, null, null, null)
+            : authentication.verifyRequest(exchange.getRequestMethod(),
+                exchange.getRequestURI(), exchange.getRequestHeaders());
+        String path = SigV4.decode(exchange.getRequestURI().getRawPath());
+        Map<String, String> query = query(verified.applicationQuery());
+        if (path.equals(CAPABILITIES_PATH)) {
+            requireOwner(verified.principal());
+            if (!exchange.getRequestMethod().equals("GET")) unsupported("Capability operation");
+            if (!query.isEmpty())
+                throw new StoreException(400, "InvalidArgument", "Capability request has unsupported query parameters");
+            requireEmptyBody(exchange, verified.payload());
+            capabilities(exchange);
+            return;
+        }
+        if (path.equals("/")) {
+            requireOwner(verified.principal());
+            if (!exchange.getRequestMethod().equals("GET") ||
+                !(query.isEmpty() || query.size() == 1 && "ListBuckets".equals(query.get("x-id"))))
+                unsupported("Service operation");
+            requireEmptyBody(exchange, verified.payload());
+            listBuckets(exchange);
+            return;
+        }
+        int slash = path.indexOf('/', 1);
+        String requestedBucket = slash < 0 ? path.substring(1) : path.substring(1, slash);
+        if (requestedBucket.isEmpty()) throw new StoreException(404, "NoSuchBucket", "Bucket not found");
+        if (slash < 0 || slash == path.length() - 1) {
+            handleBucket(exchange, query, verified.payload(), requestedBucket, verified.principal());
+        } else {
+            store.bucket(requestedBucket);
+            handleObject(exchange, path.substring(slash + 1), query, verified, requestedBucket);
+        }
+    }
+
+    private void sendStoreError(HttpExchange exchange, StoreException error, String requestId) throws IOException {
+        if (error.status == 503 && error.code.equals("SlowDown"))
+            exchange.getResponseHeaders().set("Retry-After", "1");
+        if (anonymousRead(exchange) && error.status == 404)
+            error = new StoreException(403, "AccessDenied", "Access denied");
+        if (error.deleteMarker) {
+            exchange.getResponseHeaders().set("x-amz-delete-marker", "true");
+            exchange.getResponseHeaders().set("x-amz-version-id", error.versionId);
+            if (error.modified >= 0) exchange.getResponseHeaders().set("Last-Modified",
+                DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC)
+                    .format(Instant.ofEpochMilli(error.modified)));
+        }
+        sendError(exchange, error.status, error.code, error.getMessage(), requestId);
+    }
+
    private static boolean anonymousRead(HttpExchange exchange) {
        if (!exchange.getRequestMethod().equals("GET") &&
            !exchange.getRequestMethod().equals("HEAD")) return false;
@@ -74,10 +74,36 @@ final class SigV4 {

    private Verified verifyPresigned(String method, URI uri, Headers headers) {
        if (headers.containsKey("authorization")) denied("Use one authentication method");
+        PresignedQuery query = presignedQuery(uri.getRawQuery());
+        Map<String, String> fields = query.fields();
+        if (!fields.keySet().equals(Set.of("X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date",
+                "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Signature")) ||
+            !"AWS4-HMAC-SHA256".equals(fields.get("X-Amz-Algorithm")))
+            denied("Invalid presigned parameters");
+        String[] credential = credentialScope(fields.get("X-Amz-Credential"));
+        String date = fields.get("X-Amz-Date");
+        validatePresignedTime(date, credential[1], fields.get("X-Amz-Expires"));
+        String signedHeaders = fields.get("X-Amz-SignedHeaders");
+        String canonicalHeaders = canonicalHeaders(headers, signedHeaders, Set.of("host"));
+        String scope = String.join("/", Arrays.copyOfRange(credential, 1, 5));
+        String canonical = method + "\n" + encode(decode(uri.getRawPath()), true) + "\n"
+            + canonicalQuery(query.signed()) + "\n" + canonicalHeaders + "\n"
+            + signedHeaders + "\nUNSIGNED-PAYLOAD";
+        String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
+            + hex(hash(canonical.getBytes(StandardCharsets.UTF_8)));
+        String signature = fields.get("X-Amz-Signature");
+        byte[] key = signingKey(secret(credential[0]), credential[1], region);
+        if (!HEX.matcher(signature).matches() ||
+            !MessageDigest.isEqual(hmac(key, toSign), HexFormat.of().parseHex(signature)))
+            denied("Signature mismatch");
+        return new Verified("UNSIGNED-PAYLOAD", query.application(), key, date, scope, signature, credential[0]);
+    }
+
+    private static PresignedQuery presignedQuery(String rawQuery) {
        Map<String, String> fields = new TreeMap<>();
        StringBuilder application = new StringBuilder();
        StringBuilder signed = new StringBuilder();
-        for (String part : uri.getRawQuery().split("&", -1)) {
+        for (String part : rawQuery.split("&", -1)) {
            String[] pair = part.split("=", 2);
            String name = decode(pair[0]);
            String value = decode(pair.length == 2 ? pair[1] : "");
@@ -89,17 +115,19 @@ final class SigV4 {
                appendQuery(signed, part);
            }
        }
-        if (!fields.keySet().equals(Set.of("X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date",
-                "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Signature")) ||
-            !"AWS4-HMAC-SHA256".equals(fields.get("X-Amz-Algorithm")))
-            denied("Invalid presigned parameters");
-        String[] credential = credentialScope(fields.get("X-Amz-Credential"));
-        String date = fields.get("X-Amz-Date");
-        if (!date.matches("[0-9]{8}T[0-9]{6}Z") || !date.startsWith(credential[1]))
+        return new PresignedQuery(fields, application.toString(), signed.toString());
+    }
+
+    private void validatePresignedTime(String date, String scopeDate, String rawExpires) {
+        if (!date.matches("[0-9]{8}T[0-9]{6}Z") || !date.startsWith(scopeDate))
            denied("Invalid signing date");
        long expires;
-        try { expires = Long.parseLong(fields.get("X-Amz-Expires")); }
-        catch (NumberFormatException error) { denied("Invalid presigned expiry"); return null; }
+        try {
+            expires = Long.parseLong(rawExpires);
+        } catch (NumberFormatException error) {
+            denied("Invalid presigned expiry");
+            return;
+        }
        if (expires < 1 || expires > 604800) denied("Invalid presigned expiry");
        try {
            Instant start = Instant.from(DATE.parse(date));
@@ -107,22 +135,10 @@ final class SigV4 {
            if (now.isBefore(start.minus(Duration.ofMinutes(5))) || now.isAfter(start.plusSeconds(expires)))
                denied("Presigned URL has expired or is not yet valid");
        } catch (java.time.DateTimeException error) { denied("Invalid signing date"); }
-        String signedHeaders = fields.get("X-Amz-SignedHeaders");
-        String canonicalHeaders = canonicalHeaders(headers, signedHeaders, Set.of("host"));
-        String scope = String.join("/", Arrays.copyOfRange(credential, 1, 5));
-        String canonical = method + "\n" + encode(decode(uri.getRawPath()), true) + "\n"
-            + canonicalQuery(signed.toString()) + "\n" + canonicalHeaders + "\n"
-            + signedHeaders + "\nUNSIGNED-PAYLOAD";
-        String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
-            + hex(hash(canonical.getBytes(StandardCharsets.UTF_8)));
-        String signature = fields.get("X-Amz-Signature");
-        byte[] key = signingKey(secret(credential[0]), credential[1], region);
-        if (!HEX.matcher(signature).matches() ||
-            !MessageDigest.isEqual(hmac(key, toSign), HexFormat.of().parseHex(signature)))
-            denied("Signature mismatch");
-        return new Verified("UNSIGNED-PAYLOAD", application.toString(), key, date, scope, signature, credential[0]);
    }

+    private record PresignedQuery(Map<String, String> fields, String application, String signed) { }
+
    private static boolean hasPresignedQuery(String raw) {
        return raw != null && (raw.startsWith("X-Amz-Algorithm=") || raw.contains("&X-Amz-Algorithm="));
    }