From 350611d5623a25ba8efda11d566e91f315c4df94 Mon Sep 17 00:00:00 2001
From: LunarSkyOSS <support@lunarsky.cloud>
Date: Sat, 10 Oct 2026 08:19:32 +0200
Subject: [PATCH] Handle unavailable cluster nodes explicitly

---
 src/cloud/lunarsky/store/ClusterStore.java | 29 +++++++++++++++-------
 src/cloud/lunarsky/store/NodeClient.java   | 16 +++++++-----
 src/cloud/lunarsky/store/NodeRegistry.java |  4 +--
 3 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/src/cloud/lunarsky/store/ClusterStore.java b/src/cloud/lunarsky/store/ClusterStore.java
index be5130f..c878328 100644
--- a/src/cloud/lunarsky/store/ClusterStore.java
+++ b/src/cloud/lunarsky/store/ClusterStore.java
@@ -400,23 +400,21 @@ final class ClusterStore implements ObjectStorage {
                         for (UUID id : segment.replicas()) {
                             int node = nodes.index(id);
                             if (node < 0) continue;
-                            try {
-                                byte[] candidate = nodes.get(node, segment.id(), segment.length(), segment.hash());
-                                if (copy == null) copy = candidate;
-                                healthy.add(id);
-                                healthyHosts.add(nodes.faultDomain(node, testNodeDomains));
-                            } catch (IOException error) { }
+                            byte[] candidate = readableReplica(node, segment);
+                            if (candidate == null) continue;
+                            if (copy == null) copy = candidate;
+                            healthy.add(id);
+                            healthyHosts.add(nodes.faultDomain(node, testNodeDomains));
                         }
                         if (copy == null) { unrecoverable++; continue; }
                         for (int node : PlacementPolicy.candidates(segment.id(), nodes, testNodeDomains)) {
                             UUID host = nodes.faultDomain(node, testNodeDomains);
                             if (healthyHosts.contains(host)) continue;
-                            try {
-                                nodes.repair(node, segment.id(), copy, segment.hash());
+                            if (repairReplica(node, segment, copy)) {
                                 healthy.add(nodes.node(node).id());
                                 healthyHosts.add(host);
                                 restored++;
-                            } catch (IOException error) { }
+                            }
                             if (healthyHosts.size() == 3) break;
                         }
                         if (healthyHosts.size() < 3) underReplicated++;
@@ -439,6 +437,19 @@ final class ClusterStore implements ObjectStorage {
         } catch (SQLException error) { throw databaseError(error); }
         return new RepairReport(scanned, restored, underReplicated, unrecoverable);
     }
+
+    private byte[] readableReplica(int node, Segment segment) {
+        try { return nodes.get(node, segment.id(), segment.length(), segment.hash()); }
+        catch (IOException unavailable) { return null; }
+    }
+
+    private boolean repairReplica(int node, Segment segment, byte[] copy) {
+        try {
+            nodes.repair(node, segment.id(), copy, segment.hash());
+            return true;
+        } catch (IOException unavailable) { return false; }
+    }
+
     @Override public void close() {}
 
     private final class SegmentStream extends InputStream {
diff --git a/src/cloud/lunarsky/store/NodeClient.java b/src/cloud/lunarsky/store/NodeClient.java
index 9c7c611..952f254 100644
--- a/src/cloud/lunarsky/store/NodeClient.java
+++ b/src/cloud/lunarsky/store/NodeClient.java
@@ -74,6 +74,11 @@ final class NodeClient {
         }
     }
 
+    static NodeIdentity probeIfAvailable(URI url, String token) {
+        try { return probe(url, token); }
+        catch (IOException offline) { return null; }
+    }
+
     static void validateUrl(URI url) {
         if (url == null || !"http".equals(url.getScheme()) || url.getHost() == null ||
             url.getPort() < 1 || url.getRawUserInfo() != null ||
@@ -86,12 +91,11 @@ final class NodeClient {
         Set<UUID> healthy = new HashSet<>();
         for (int i = 0; i < nodes.size(); i++) {
             Node node = nodes.get(i);
-            try {
-                NodeIdentity actual = probe(node.url(), token);
-                if (actual.nodeId().equals(node.id()) && actual.hostId().equals(node.hostId()))
-                    healthy.add(faultDomain(i, testNodeDomains));
-                if (healthy.size() >= required) return true;
-            } catch (IOException error) { }
+            NodeIdentity actual = probeIfAvailable(node.url(), token);
+            if (actual == null) continue;
+            if (actual.nodeId().equals(node.id()) && actual.hostId().equals(node.hostId()))
+                healthy.add(faultDomain(i, testNodeDomains));
+            if (healthy.size() >= required) return true;
         }
         return false;
     }
diff --git a/src/cloud/lunarsky/store/NodeRegistry.java b/src/cloud/lunarsky/store/NodeRegistry.java
index b6cfc4f..931a223 100644
--- a/src/cloud/lunarsky/store/NodeRegistry.java
+++ b/src/cloud/lunarsky/store/NodeRegistry.java
@@ -126,9 +126,7 @@ final class NodeRegistry {
         for (URI url : urls) {
             NodeClient.Node node = stored.get(url.toString());
             if (node == null) throw new IOException("Unregistered storage node URL: " + url);
-            NodeIdentity actual = null;
-            try { actual = NodeClient.probe(url, token); }
-            catch (IOException offline) { }
+            NodeIdentity actual = NodeClient.probeIfAvailable(url, token);
             if (actual != null && (!actual.nodeId().equals(node.id()) || !actual.hostId().equals(node.hostId())))
                 throw new IOException("Storage node identity changed at " + url);
             configured.add(node);