From a71c4fa5f1dc57dc9c4efe377e39ac238a8674d1 Mon Sep 17 00:00:00 2001 From: LunarSkyOSS Date: Sat, 10 Oct 2026 23:57:36 +0200 Subject: [PATCH] Support TLS for cluster node transport --- Dockerfile | 1 + README.md | 21 +++- TESTS.md | 3 +- compose.cluster.tls.yaml | 58 ++++++++++ scripts/test.sh | 1 + src/cloud/lunarsky/store/ClusterNode.java | 7 +- src/cloud/lunarsky/store/ClusterTls.java | 99 ++++++++++++++++ src/cloud/lunarsky/store/NodeClient.java | 43 +++++-- test/cloud/lunarsky/store/ClusterTlsTest.java | 107 ++++++++++++++++++ 9 files changed, 327 insertions(+), 13 deletions(-) create mode 100644 compose.cluster.tls.yaml create mode 100644 src/cloud/lunarsky/store/ClusterTls.java create mode 100644 test/cloud/lunarsky/store/ClusterTlsTest.java diff --git a/Dockerfile b/Dockerfile index 4694a96..a7f7a65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ RUN java --add-modules jdk.httpserver -cp /out:/tmp/hash4j.jar cloud.lunarsky.st RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.HttpTest RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClientLimitsTest RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClusterNodeTest +RUN java --add-modules jdk.httpserver,java.net.http -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.ClusterTlsTest RUN java -cp /out:/tmp/hash4j.jar cloud.lunarsky.store.CliTest FROM eclipse-temurin:21-jre-alpine diff --git a/README.md b/README.md index 37a2117..584a5b9 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Source: [GitHub](https://github.com/LunarSkyOSS/ObjectStore) · [Gitea mirror](h - [Capability discovery](#capability-discovery) - [Java client](#java-client) - [Local cluster prototype](#local-cluster-prototype) +- [Node transport TLS](#node-transport-tls) - [Migrating a local cluster](#migrating-a-local-cluster) - [Adding a cluster node](#adding-a-cluster-node) - [Cluster maintenance and recovery](#cluster-maintenance-and-recovery) @@ -36,6 +37,7 @@ ObjectStore creates the configured default bucket at startup. Additional buckets - ✅ Configurable per-object and total logical size limits - ✅ CLI status, version, and full payload verification - ✅ Local cluster prototype with stable node IDs and host-aware placement code +- ✅ Optional HTTPS between cluster processes and storage nodes - ✅ Opt-in automatic repair, rebalance, and guarded garbage collection in the local cluster - ✅ Metadata backup and tested restore to a separate local PostgreSQL instance - ✅ Manual [two-machine durability and metadata-restore drill](tests/two-host/README.md) @@ -127,6 +129,23 @@ docker compose --env-file /path/to/cluster.env -f compose.cluster.yaml run --rm The cluster S3 endpoint binds to `127.0.0.1:9001`; storage nodes and PostgreSQL have no published ports. The separate repair container holds the repair credential and restores missing or corrupt replicas. +## Node transport TLS + +The default local Compose cluster uses HTTP inside its private Docker network. For an HTTPS test, give each node a PKCS#12 keystore containing its private key and a certificate whose DNS subject alternative name matches its `CLUSTER_NODES` hostname. Give the gateway, repair, garbage collection, and maintenance processes a PKCS#12 truststore containing the issuing CA or each node certificate. Mount the files read-only and keep the keystores and password files outside Git. + +Set `NODE_TLS_KEYSTORE` and `NODE_TLS_PASSWORD_FILE` on each node. Set `CLUSTER_TLS_TRUSTSTORE` and `CLUSTER_TLS_PASSWORD_FILE` on every process that contacts nodes, and change each node URL to `https://`. With a truststore configured, HTTP node URLs are rejected. The client verifies the certificate chain and hostname; a failed handshake does not fall back to HTTP. Both settings in each pair are required. Restart affected processes after rotating certificates or truststores. + +The optional [Compose TLS overlay](compose.cluster.tls.yaml) expects `node-a.p12` through `node-d.p12`, matching `.pass` files, and `trust.p12` with `trust.pass` in `CLUSTER_TLS_DIR`. Set that variable to a private certificate directory and include both Compose files: + +```sh +CLUSTER_TLS_DIR=/private/objectstore-certs docker compose --env-file /path/to/cluster.env \ + -f compose.cluster.yaml -f compose.cluster.tls.yaml up -d --build +``` + +The files must be readable by container UID 10001 without making private keys or passwords world-readable. Add HTTPS URLs for additional nodes when expanding the cluster. + +This secures node traffic only. The local cluster still lacks automatic PostgreSQL failover, database TLS configuration, encryption at rest, and production multi-server validation. Its HTTP S3 gateway remains bound to localhost; use a separate trusted proxy for external TLS. Do not treat the TLS overlay as a production deployment. + Multipart parts are stored on cluster nodes and indexed in PostgreSQL. Incomplete uploads count toward the logical capacity limit; abort them to release that capacity. Repair includes staged parts. The gateway upgrades the metadata schema when it starts, so back up the database before upgrading an existing cluster. Node UUIDs persist on their volumes, and replica manifests use those UUIDs so reordering configured URLs cannot move an existing replica. Each node also has an operator-assigned physical host UUID. New writes require acknowledgements from two different host UUIDs. The optional `CLUSTER_TEST_NODE_DOMAINS=true` override counts containers instead, solely for local process tests; all containers in this Compose file share one physical host. @@ -169,7 +188,7 @@ ObjectStore uses the socket peer as the client IP and ignores forwarded-IP heade Enable these limits only on a deliberately public endpoint. They also apply to direct localhost storage calls to that same endpoint; leave them disabled for the local-only setup or run a separate local-only instance if local storage calls must be exempt. A direct loopback `/health` probe without a forwarded-IP header remains exempt. The byte limit is aggregate ingress plus egress for each IP, and several users behind one NAT share it. If a proxy buffers complete uploads before forwarding them, the upload byte limit controls proxy-to-ObjectStore traffic, not the client's initial upload speed; disable request buffering when end-to-end upload pacing is required. It is a fairness control, not a defense against connection floods before the Java handler runs. Put an internet-facing proxy or firewall in front of the gateway for TLS, connection limits, request timeouts, and buffering controls. Do not expose storage nodes or PostgreSQL publicly. -The local cluster has no automatic metadata failover, private-network TLS, scoped credentials, or physical host verification. Garbage collection can remove data required by an older metadata backup, so its retention guard is essential. Host UUIDs are operator labels, not proof that machines have separate power, disks, or network paths. Keep `CLUSTER_LOCAL_DEV=true` limited to local tests. +The default local cluster still uses plaintext node and PostgreSQL connections. The optional TLS overlay secures node traffic, but PostgreSQL TLS, automatic metadata failover, scoped credentials, and physical host verification remain absent. Garbage collection can remove data required by an older metadata backup, so its retention guard is essential. Host UUIDs are operator labels, not proof that machines have separate power, disks, or network paths. Keep `CLUSTER_LOCAL_DEV=true` limited to local tests. The standalone cluster node binds to localhost by default. Set `NODE_BIND` only for a private test network; the Compose file binds inside its private Docker network. PostgreSQL JDBC 42.7.14 is bundled in the image with its license inside the JAR. diff --git a/TESTS.md b/TESTS.md index dad421a..77ed099 100644 --- a/TESTS.md +++ b/TESTS.md @@ -21,10 +21,11 @@ The script compiles the source and test programs into `out/classes`, then runs: | `HttpTest` | Signed capability discovery, presigned URLs, streaming uploads and trailers, object and bucket operations, ACL grants with a second access key and public reads, copies, checksum persistence and rejection, ranges, listing, metadata, tags, multipart uploads, and versioning in single-node mode. | | `ClientLimitsTest` | Disabled defaults, trusted-proxy address validation, ignored untrusted headers, per-IP request refusal, and paced response bytes. | | `ClusterNodeTest` | Node identity and locking, authenticated segment transfers, checksum rejection, repair authorization, inventory and guarded deletion, and restart cleanup. | +| `ClusterTlsTest` | HTTPS node identity and segment roundtrip with a trusted certificate, plus rejection of untrusted and wrong-host certificates. | | `CliTest` | Version, status, verification, and a nonzero result for corrupt data. | | `ClientTest` and `MultipartClientTest` | Java client request signing, capability discovery, error handling, and multipart operations. | -The script exits nonzero on failure. The test programs use temporary local directories and loopback HTTP ports; they do not use an existing ObjectStore volume. +The script exits nonzero on failure. The test programs use temporary local directories and loopback HTTP or HTTPS ports; they do not use an existing ObjectStore volume. `ClusterTlsTest` uses the JDK's `keytool` to create disposable test certificates. ## Disposable Docker cluster tests diff --git a/compose.cluster.tls.yaml b/compose.cluster.tls.yaml new file mode 100644 index 0000000..6c1f16e --- /dev/null +++ b/compose.cluster.tls.yaml @@ -0,0 +1,58 @@ +x-client-tls: &client-tls + CLUSTER_NODES: https://node-a:9100,https://node-b:9100,https://node-c:9100 + CLUSTER_TLS_TRUSTSTORE: /run/objectstore-tls/trust.p12 + CLUSTER_TLS_PASSWORD_FILE: /run/objectstore-tls/trust.pass + +x-tls-volume: &tls-volume + - ${CLUSTER_TLS_DIR:?Set CLUSTER_TLS_DIR to a private certificate directory}:/run/objectstore-tls:ro + +x-node-health: &node-health + test: ["CMD", "nc", "-z", "-w", "2", "127.0.0.1", "9100"] + interval: 10s + timeout: 3s + retries: 3 + +services: + gateway: + environment: *client-tls + volumes: *tls-volume + + repair: + environment: *client-tls + volumes: *tls-volume + + gc: + environment: *client-tls + volumes: *tls-volume + + maintenance: + environment: *client-tls + volumes: *tls-volume + + node-a: + environment: + NODE_TLS_KEYSTORE: /run/objectstore-tls/node-a.p12 + NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-a.pass + volumes: *tls-volume + healthcheck: *node-health + + node-b: + environment: + NODE_TLS_KEYSTORE: /run/objectstore-tls/node-b.p12 + NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-b.pass + volumes: *tls-volume + healthcheck: *node-health + + node-c: + environment: + NODE_TLS_KEYSTORE: /run/objectstore-tls/node-c.p12 + NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-c.pass + volumes: *tls-volume + healthcheck: *node-health + + node-d: + environment: + NODE_TLS_KEYSTORE: /run/objectstore-tls/node-d.p12 + NODE_TLS_PASSWORD_FILE: /run/objectstore-tls/node-d.pass + volumes: *tls-volume + healthcheck: *node-health diff --git a/scripts/test.sh b/scripts/test.sh index 2debe09..76b42a7 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -9,5 +9,6 @@ java --add-modules jdk.httpserver -cp out/classes:lib/hash4j-0.30.0.jar cloud.lu java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.HttpTest java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClientLimitsTest java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClusterNodeTest +java --add-modules jdk.httpserver,java.net.http -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.ClusterTlsTest java -cp out/classes:lib/hash4j-0.30.0.jar cloud.lunarsky.store.CliTest bash client/scripts/test.sh diff --git a/src/cloud/lunarsky/store/ClusterNode.java b/src/cloud/lunarsky/store/ClusterNode.java index 540f1ba..9acab3e 100644 --- a/src/cloud/lunarsky/store/ClusterNode.java +++ b/src/cloud/lunarsky/store/ClusterNode.java @@ -1,7 +1,6 @@ package cloud.lunarsky.store; import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpServer; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -325,7 +324,8 @@ public final class ClusterNode implements AutoCloseable { env.get("CLUSTER_REPAIR_TOKEN"), UUID.fromString(env.get("CLUSTER_HOST_ID"))); int port = Integer.parseInt(env.getOrDefault("NODE_PORT", "9100")); - var server = HttpServer.create(new InetSocketAddress(env.getOrDefault("NODE_BIND", "127.0.0.1"), port), 64); + var server = ClusterTls.nodeServer( + new InetSocketAddress(env.getOrDefault("NODE_BIND", "127.0.0.1"), port), env); var executor = Executors.newVirtualThreadPerTaskExecutor(); server.setExecutor(executor); server.createContext("/", node::handle); @@ -336,6 +336,7 @@ public final class ClusterNode implements AutoCloseable { catch (IOException error) { System.err.println("Node close failed: " + error); } })); server.start(); - System.out.println("ObjectStore cluster node listening on :" + port); + System.out.println("ObjectStore cluster node listening on :" + port + + (server instanceof com.sun.net.httpserver.HttpsServer ? " (TLS)" : " (HTTP)")); } } diff --git a/src/cloud/lunarsky/store/ClusterTls.java b/src/cloud/lunarsky/store/ClusterTls.java new file mode 100644 index 0000000..8cdf8f1 --- /dev/null +++ b/src/cloud/lunarsky/store/ClusterTls.java @@ -0,0 +1,99 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; + +final class ClusterTls { + private ClusterTls() { } + + static HttpServer nodeServer(InetSocketAddress address, Map env) throws IOException { + String keyStore = env.get("NODE_TLS_KEYSTORE"); + String passwordFile = env.get("NODE_TLS_PASSWORD_FILE"); + if (missing(keyStore) && missing(passwordFile)) return HttpServer.create(address, 64); + if (missing(keyStore) || missing(passwordFile)) + throw new IOException("Node TLS requires both NODE_TLS_KEYSTORE and NODE_TLS_PASSWORD_FILE"); + SSLContext context = serverContext(Path.of(keyStore), Path.of(passwordFile)); + HttpsServer server = HttpsServer.create(address, 64); + server.setHttpsConfigurator(new HttpsConfigurator(context)); + return server; + } + + static HttpClient client(Map env, Duration timeout) throws IOException { + String trustStore = env.get("CLUSTER_TLS_TRUSTSTORE"); + String passwordFile = env.get("CLUSTER_TLS_PASSWORD_FILE"); + if (missing(trustStore) != missing(passwordFile)) + throw new IOException("Cluster TLS requires both CLUSTER_TLS_TRUSTSTORE and CLUSTER_TLS_PASSWORD_FILE"); + HttpClient.Builder builder = HttpClient.newBuilder().connectTimeout(timeout); + if (!missing(trustStore)) + builder.sslContext(clientContext(Path.of(trustStore), Path.of(passwordFile))); + return builder.build(); + } + + private static SSLContext serverContext(Path keyStore, Path passwordFile) throws IOException { + char[] password = password(passwordFile); + try { + KeyStore keys = load(keyStore, password); + KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + managers.init(keys, password); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(managers.getKeyManagers(), null, null); + return context; + } catch (GeneralSecurityException error) { + throw new IOException("Could not configure node TLS", error); + } finally { + Arrays.fill(password, '\0'); + } + } + + private static SSLContext clientContext(Path trustStore, Path passwordFile) throws IOException { + char[] password = password(passwordFile); + try { + KeyStore trust = load(trustStore, password); + TrustManagerFactory managers = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + managers.init(trust); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, managers.getTrustManagers(), null); + return context; + } catch (GeneralSecurityException error) { + throw new IOException("Could not configure cluster TLS trust", error); + } finally { + Arrays.fill(password, '\0'); + } + } + + private static KeyStore load(Path path, char[] password) throws IOException, GeneralSecurityException { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (InputStream input = Files.newInputStream(path)) { + store.load(input, password); + } + return store; + } + + private static char[] password(Path file) throws IOException { + String value = Files.readString(file, StandardCharsets.UTF_8); + if (value.endsWith("\n")) value = value.substring(0, value.length() - 1); + if (value.endsWith("\r")) value = value.substring(0, value.length() - 1); + if (value.isEmpty()) throw new IOException("Cluster TLS password file is empty"); + return value.toCharArray(); + } + + private static boolean missing(String value) { + return value == null || value.isBlank(); + } +} diff --git a/src/cloud/lunarsky/store/NodeClient.java b/src/cloud/lunarsky/store/NodeClient.java index a8a9722..4673c57 100644 --- a/src/cloud/lunarsky/store/NodeClient.java +++ b/src/cloud/lunarsky/store/NodeClient.java @@ -21,18 +21,21 @@ final class NodeClient { record Node(UUID id, UUID hostId, URI url) {} record StoredSegment(UUID id, long modified) {} - private static final HttpClient IDENTITY_HTTP = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(2)).build(); + private static HttpClient identityHttp; private final List nodes; private final String token; private final String repairToken; - private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(3)).build(); + private final HttpClient http; private final ConcurrentHashMap unreadableUntil = new ConcurrentHashMap<>(); private final ConcurrentHashMap healthyUntil = new ConcurrentHashMap<>(); private static final long READ_RETRY_NANOS = TimeUnit.SECONDS.toNanos(5); private static final long HEALTH_FRESH_NANOS = TimeUnit.SECONDS.toNanos(3); - NodeClient(List nodes, String token, String repairToken) { + NodeClient(List nodes, String token, String repairToken) throws IOException { + this(nodes, token, repairToken, ClusterTls.client(System.getenv(), Duration.ofSeconds(3))); + } + + NodeClient(List nodes, String token, String repairToken, HttpClient http) { if (nodes.isEmpty() || nodes.stream().map(Node::id).distinct().count() != nodes.size() || nodes.stream().map(Node::url).distinct().count() != nodes.size()) throw new IllegalArgumentException("Cluster node IDs and URLs must be unique"); @@ -45,6 +48,7 @@ final class NodeClient { this.nodes = List.copyOf(nodes); this.token = token; this.repairToken = repairToken; + this.http = java.util.Objects.requireNonNull(http); } int count() { return nodes.size(); } @@ -63,12 +67,16 @@ final class NodeClient { } static NodeIdentity probe(URI url, String token) throws IOException { + return probe(url, token, identityHttp()); + } + + static NodeIdentity probe(URI url, String token, HttpClient http) throws IOException { validateUrl(url); if (token == null || token.length() < 32) throw new IllegalArgumentException("Invalid cluster token"); HttpRequest request = HttpRequest.newBuilder(url.resolve("/identity")) .timeout(Duration.ofSeconds(2)).header("X-Cluster-Token", token).GET().build(); try { - HttpResponse response = IDENTITY_HTTP.send(request, HttpResponse.BodyHandlers.ofInputStream()); + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); try (InputStream body = response.body()) { if (response.statusCode() != 200) throw new IOException("Node identity request failed: " + response.statusCode()); byte[] bytes = body.readNBytes(128); @@ -90,19 +98,38 @@ final class NodeClient { catch (IOException offline) { return null; } } + private static NodeIdentity probeIfAvailable(URI url, String token, HttpClient http) { + try { return probe(url, token, http); } + catch (IOException offline) { return null; } + } + + private static synchronized HttpClient identityHttp() throws IOException { + if (identityHttp == null) + identityHttp = ClusterTls.client(System.getenv(), Duration.ofSeconds(2)); + return identityHttp; + } + static void validateUrl(URI url) { - if (url == null || !"http".equals(url.getScheme()) || url.getHost() == null || + String trustStore = System.getenv("CLUSTER_TLS_TRUSTSTORE"); + validateUrl(url, trustStore != null && !trustStore.isBlank()); + } + + static void validateUrl(URI url, boolean requireHttps) { + if (url == null || !("http".equals(url.getScheme()) || "https".equals(url.getScheme())) || + url.getHost() == null || url.getPort() < 1 || url.getRawUserInfo() != null || (url.getRawPath() != null && !url.getRawPath().isEmpty()) || url.getRawQuery() != null || url.getRawFragment() != null) throw new IllegalArgumentException("Invalid private storage node URL"); + if (requireHttps && !"https".equals(url.getScheme())) + throw new IllegalArgumentException("Cluster TLS truststore requires HTTPS node URLs"); } boolean availableHostsAtLeast(int required, boolean testNodeDomains) { Set healthy = new HashSet<>(); for (int i = 0; i < nodes.size(); i++) { Node node = nodes.get(i); - NodeIdentity actual = probeIfAvailable(node.url(), token); + NodeIdentity actual = probeIfAvailable(node.url(), token, http); if (actual == null || !actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())) { markUnreadable(node); continue; @@ -159,7 +186,7 @@ final class NodeClient { Node node = nodes.get(index); if (unreadable(node)) throw new IOException("Storage node is temporarily unreachable"); if (!recentlyHealthy(node)) { - NodeIdentity actual = probeIfAvailable(node.url(), token); + NodeIdentity actual = probeIfAvailable(node.url(), token, http); if (actual == null || !actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())) { markUnreadable(node); throw new IOException("Storage node is temporarily unreachable"); diff --git a/test/cloud/lunarsky/store/ClusterTlsTest.java b/test/cloud/lunarsky/store/ClusterTlsTest.java new file mode 100644 index 0000000..6f8cd6c --- /dev/null +++ b/test/cloud/lunarsky/store/ClusterTlsTest.java @@ -0,0 +1,107 @@ +package cloud.lunarsky.store; + +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsServer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public final class ClusterTlsTest { + private static final String PASSWORD = "local-test-password-0123456789"; + + public static void main(String[] args) throws Exception { + Path directory = Files.createTempDirectory("objectstore-tls-"); + Path keyStore = directory.resolve("node.p12"); + Path trustStore = directory.resolve("trust.p12"); + Path certificate = directory.resolve("node.crt"); + Path passwordFile = directory.resolve("password"); + Files.writeString(passwordFile, PASSWORD + "\n"); + String keytool = Path.of(System.getProperty("java.home"), "bin", "keytool").toString(); + run(keytool, "-genkeypair", "-alias", "node", "-keyalg", "RSA", "-keysize", "2048", + "-validity", "2", "-dname", "CN=localhost", "-ext", "SAN=DNS:localhost", + "-storetype", "PKCS12", "-keystore", keyStore.toString(), "-storepass", PASSWORD, + "-keypass", PASSWORD, "-noprompt"); + run(keytool, "-exportcert", "-alias", "node", "-keystore", keyStore.toString(), + "-storepass", PASSWORD, "-file", certificate.toString()); + run(keytool, "-importcert", "-alias", "node", "-file", certificate.toString(), + "-keystore", trustStore.toString(), "-storetype", "PKCS12", "-storepass", PASSWORD, + "-noprompt"); + + Map serverConfig = Map.of( + "NODE_TLS_KEYSTORE", keyStore.toString(), "NODE_TLS_PASSWORD_FILE", passwordFile.toString()); + Map clientConfig = Map.of( + "CLUSTER_TLS_TRUSTSTORE", trustStore.toString(), + "CLUSTER_TLS_PASSWORD_FILE", passwordFile.toString()); + String token = "tls-test-cluster-token-0123456789"; + String repairToken = "tls-test-repair-token-0123456789"; + UUID hostId = UUID.randomUUID(); + byte[] data = "encrypted transport".getBytes(java.nio.charset.StandardCharsets.UTF_8); + try (ClusterNode node = new ClusterNode(directory.resolve("data"), token, repairToken, hostId)) { + HttpServer server = ClusterTls.nodeServer(new InetSocketAddress("127.0.0.1", 0), serverConfig); + require(server instanceof HttpsServer, "Node did not enable HTTPS"); + server.createContext("/", node::handle); + server.start(); + try { + URI url = URI.create("https://localhost:" + server.getAddress().getPort()); + HttpClient trusted = ClusterTls.client(clientConfig, Duration.ofSeconds(3)); + NodeIdentity identity = NodeClient.probe(url, token, trusted); + require(identity.hostId().equals(hostId), "TLS probe returned wrong node identity"); + NodeClient client = new NodeClient(List.of( + new NodeClient.Node(identity.nodeId(), hostId, url)), token, repairToken, trusted); + UUID segment = UUID.randomUUID(); + client.put(0, segment, data, SigV4.hash(data)); + require(java.util.Arrays.equals(data, client.get(0, segment, data.length, SigV4.hash(data))), + "TLS segment roundtrip failed"); + HttpRequest request = HttpRequest.newBuilder(url.resolve("/identity")) + .header("X-Cluster-Token", token).GET().build(); + try { + ClusterTls.client(Map.of(), Duration.ofSeconds(3)) + .send(request, HttpResponse.BodyHandlers.discarding()); + throw new AssertionError("Untrusted certificate was accepted"); + } catch (IOException expected) { } + URI wrongHost = URI.create("https://127.0.0.1:" + server.getAddress().getPort()); + try { + NodeClient.probe(wrongHost, token, trusted); + throw new AssertionError("Wrong certificate hostname was accepted"); + } catch (IOException expected) { } + } finally { + server.stop(0); + } + } + try { + ClusterTls.nodeServer(new InetSocketAddress("127.0.0.1", 0), + Map.of("NODE_TLS_KEYSTORE", keyStore.toString())); + throw new AssertionError("Incomplete TLS configuration was accepted"); + } catch (IOException expected) { } + try { + ClusterTls.client(Map.of("CLUSTER_TLS_TRUSTSTORE", trustStore.toString()), + Duration.ofSeconds(3)); + throw new AssertionError("Incomplete cluster trust configuration was accepted"); + } catch (IOException expected) { } + try { + NodeClient.validateUrl(URI.create("http://localhost:9100"), true); + throw new AssertionError("HTTP node URL was accepted with cluster TLS enabled"); + } catch (IllegalArgumentException expected) { } + System.out.println("Cluster TLS tests passed: trusted roundtrip, untrusted and hostname rejection, no HTTP downgrade"); + } + + private static void run(String... command) throws Exception { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + if (process.waitFor() != 0) throw new AssertionError("keytool failed: " + output); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +}