admin/ObjectStore
1
1 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Release ObjectStore 0.0.4

Browse Source
This commit is contained in:
admin committed 2026-10-10 10:01:43 +02:00
1 parent 00d2af2fb0
commit ed7712a8af
19 files changed
+1025 -114
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/cloud/lunarsky/store/ClusterIntegrationTest.java
+39
View File
@@ -8,6 +8,9 @@ import java.util.Arrays;
import java.util.HexFormat;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import java.util.UUID;
public final class ClusterIntegrationTest {
private static final String KEY = "cluster-test/survivor";
@@ -200,6 +203,42 @@ public final class ClusterIntegrationTest {
}
System.out.println("Joined node accepted new placements while previous objects stayed readable");
}
case "verify-expanded" -> {
for (int i = 0; i < 32; i++) {
try (var opened = store.open(bucket, "cluster-test/expanded-" + i)) {
require(("expanded object " + i).equals(new String(opened.stream().readAllBytes(),
StandardCharsets.UTF_8)), "Expanded object was lost during maintenance");
}
}
System.out.println("Expanded objects survived repair and cleanup");
}
case "balanced" -> {
try (var connection = java.sql.DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"))) {
NodeClient nodes = NodeRegistry.load(connection,
Arrays.stream(urls).map(URI::create).toList(), env.get("CLUSTER_TOKEN"), null);
int checked = 0;
try (var query = connection.createStatement();
var result = query.executeQuery("SELECT s.segment_id, s.replica_ids FROM cluster_segments s " +
"JOIN cluster_objects o ON o.generation=s.generation")) {
while (result.next()) {
UUID segment = (UUID) result.getObject(1);
Set<UUID> preferred = new HashSet<>();
Set<UUID> hosts = new HashSet<>();
for (int index : PlacementPolicy.candidates(segment, nodes, true)) {
if (hosts.add(nodes.faultDomain(index, true))) preferred.add(nodes.node(index).id());
if (preferred.size() == 3) break;
}
Set<UUID> actual = new HashSet<>();
for (Object id : (Object[]) result.getArray(2).getArray()) actual.add((UUID) id);
require(actual.equals(preferred), "Segment did not move to preferred hosts");
checked++;
}
}
require(checked > 0, "No live segments were checked for placement");
}
System.out.println("Existing segments balanced across preferred hosts");
}
default -> throw new IllegalArgumentException("Unknown test phase");
}
}
test/cloud/lunarsky/store/ClusterNodeTest.java
+22
View File
@@ -75,6 +75,28 @@ public final class ClusterNodeTest {
client.repair(0, id, value, SigV4.hash(value));
require(java.util.Arrays.equals(value, client.get(0, id, value.length, SigV4.hash(value))),
"Repair did not restore the original bytes");
UUID orphan = UUID.randomUUID();
client.put(0, orphan, value, SigV4.hash(value));
var inventory = client.inventory(0, orphan.toString().substring(0, 2), null);
var listed = inventory.stream().filter(entry -> entry.id().equals(orphan)).findFirst().orElseThrow();
var forgedInventory = HttpRequest.newBuilder(uri.resolve("/segments?shard=" +
orphan.toString().substring(0, 2))).header("X-Cluster-Token", token)
.header("X-Cluster-Expected-Node", nodeId.toString()).GET().build();
require(HttpClient.newHttpClient().send(forgedInventory, HttpResponse.BodyHandlers.discarding())
.statusCode() == 403, "Gateway token was allowed to list node segments");
var forgedDelete = HttpRequest.newBuilder(uri.resolve("/segments/" + orphan))
.header("X-Cluster-Token", token).header("X-Cluster-Expected-Node", nodeId.toString())
.header("X-Cluster-Expected-Mtime", Long.toString(listed.modified()))
.header("X-Cluster-Gc-Min-Age-Millis", "0").DELETE().build();
require(HttpClient.newHttpClient().send(forgedDelete, HttpResponse.BodyHandlers.discarding())
.statusCode() == 403, "Gateway token was allowed to delete a segment");
require(!client.deleteOrphan(0, new NodeClient.StoredSegment(orphan, listed.modified() - 1), 0),
"Stale inventory entry deleted a segment");
require(client.deleteOrphan(0, listed, 0), "Confirmed orphan segment was not deleted");
try {
client.get(0, orphan, value.length, SigV4.hash(value));
throw new AssertionError("Deleted orphan was still readable");
} catch (IOException expected) { }
} finally { server.stop(0); }
}
Path pending = root.resolve("pending").resolve("unfinished.part");
test/cloud/lunarsky/store/HttpTest.java
+112 -1
View File
@@ -13,10 +13,14 @@ import java.time.Clock;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.Comparator;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.Executors;
import java.util.zip.CRC32;
import java.util.zip.CRC32C;
import java.util.zip.Checksum;
public final class HttpTest {
private static final String ACCESS = "TESTACCESSKEY123";
@@ -129,6 +133,106 @@ public final class HttpTest {
throw new AssertionError("Empty list page failed: " + emptyPage.body());
}
private static void testCopy(HttpClient client, String base) throws Exception {
String source = "folder/copy source.txt";
String target = "folder/copied.txt";
byte[] body = "copy source".getBytes(StandardCharsets.UTF_8);
status(200, client.send(signedUri(URI.create(base + "/objects/" + SigV4.encode(source, true)),
"PUT", body, Map.of("content-type", "text/plain")), HttpResponse.BodyHandlers.ofByteArray()));
String header = "/objects/" + SigV4.encode(source, true);
var result = client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", header)), HttpResponse.BodyHandlers.ofString());
if (result.statusCode() != 200 || !result.body().contains("<CopyObjectResult>") ||
!result.body().contains("<ETag>&quot;"))
throw new AssertionError("CopyObject failed: " + result.body());
var copied = client.send(signed(base, "GET", target, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
status(200, copied);
if (!java.util.Arrays.equals(body, copied.body()) ||
!"text/plain".equals(copied.headers().firstValue("content-type").orElse("")))
throw new AssertionError("Copied body or content type mismatch");
status(200, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", header, "x-amz-metadata-directive", "REPLACE",
"content-type", "text/markdown")), HttpResponse.BodyHandlers.ofByteArray()));
var replaced = client.send(signed(base, "GET", target, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
status(200, replaced);
if (!java.util.Arrays.equals(body, replaced.body()) ||
!"text/markdown".equals(replaced.headers().firstValue("content-type").orElse("")))
throw new AssertionError("REPLACE metadata directive failed");
status(200, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/objects/" + target)),
HttpResponse.BodyHandlers.ofByteArray()));
status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/objects/missing")),
HttpResponse.BodyHandlers.ofByteArray()));
status(404, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/other/source")),
HttpResponse.BodyHandlers.ofByteArray()));
status(400, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", "/objects/source?versionId=1")),
HttpResponse.BodyHandlers.ofByteArray()));
status(501, client.send(signedUri(URI.create(base + "/objects/" + target), "PUT",
new byte[0], Map.of("x-amz-copy-source", header, "x-amz-metadata-directive", "REPLACE",
"content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signed(base, "DELETE", source, new byte[0]),
HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signed(base, "DELETE", target, new byte[0]),
HttpResponse.BodyHandlers.ofByteArray()));
}
private static String encodedChecksum(String algorithm, byte[] body) throws Exception {
if (algorithm.startsWith("CRC")) {
Checksum checksum = algorithm.equals("CRC32") ? new CRC32() : new CRC32C();
checksum.update(body, 0, body.length);
long value = checksum.getValue();
return Base64.getEncoder().encodeToString(new byte[]{(byte) (value >>> 24),
(byte) (value >>> 16), (byte) (value >>> 8), (byte) value});
}
String name = algorithm.equals("SHA1") ? "SHA-1" :
algorithm.equals("SHA256") ? "SHA-256" :
algorithm.equals("SHA512") ? "SHA-512" : "MD5";
return Base64.getEncoder().encodeToString(java.security.MessageDigest.getInstance(name).digest(body));
}
private static void testChecksums(HttpClient client, String base) throws Exception {
URI uri = URI.create(base + "/objects/checksum-target");
byte[] body = "checksum payload".getBytes(StandardCharsets.UTF_8);
String md5 = encodedChecksum("MD5", body);
for (String algorithm : new String[]{"CRC32", "CRC32C", "SHA1", "SHA256", "SHA512", "MD5"}) {
String header = "x-amz-checksum-" + algorithm.toLowerCase(java.util.Locale.ROOT);
String checksum = encodedChecksum(algorithm, body);
var stored = client.send(signedUri(uri, "PUT", body,
Map.of("content-md5", md5, header, checksum, "x-amz-sdk-checksum-algorithm", algorithm)),
HttpResponse.BodyHandlers.ofByteArray());
status(200, stored);
if (!checksum.equals(stored.headers().firstValue(header).orElse("")))
throw new AssertionError("Missing checksum response: " + algorithm);
var bad = client.send(signedUri(uri, "PUT", body,
Map.of(header, Base64.getEncoder().encodeToString(new byte[algorithm.startsWith("CRC") ? 4 :
algorithm.equals("SHA1") ? 20 : algorithm.equals("SHA256") ? 32 :
algorithm.equals("SHA512") ? 64 : 16]))), HttpResponse.BodyHandlers.ofString());
if (bad.statusCode() != 400 || !bad.body().contains("BadDigest"))
throw new AssertionError("Mismatched " + algorithm + " accepted: " + bad.body());
var unchanged = client.send(signedUri(uri, "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray());
status(200, unchanged);
if (!java.util.Arrays.equals(body, unchanged.body()))
throw new AssertionError("Bad checksum replaced stored object");
}
var badMd5 = client.send(signedUri(uri, "PUT", body,
Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString());
if (badMd5.statusCode() != 400 || !badMd5.body().contains("BadDigest"))
throw new AssertionError("Mismatched Content-MD5 accepted");
status(400, client.send(signedUri(uri, "PUT", body,
Map.of("content-md5", "invalid")), HttpResponse.BodyHandlers.ofByteArray()));
status(400, client.send(signedUri(uri, "PUT", body,
Map.of("x-amz-checksum-crc32", encodedChecksum("CRC32", body),
"x-amz-sdk-checksum-algorithm", "CRC32C")), HttpResponse.BodyHandlers.ofByteArray()));
status(501, client.send(signedUri(uri, "PUT", body,
Map.of("x-amz-checksum-crc64nvme", "AAAAAAAAAAA=")), HttpResponse.BodyHandlers.ofByteArray()));
status(204, client.send(signedUri(uri, "DELETE", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofByteArray()));
}
private static void testMultipart(HttpClient client, String base) throws Exception {
String movie = "folder/video.mp4";
URI initiate = URI.create(base + "/objects/" + movie + "?uploads=");
@@ -144,6 +248,11 @@ public final class HttpTest {
"?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
status(200, partOne);
status(200, partTwo);
var rejectedPart = client.send(signedUri(URI.create(base + "/objects/" + movie +
"?partNumber=1&uploadId=" + upload), "PUT", first,
Map.of("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")), HttpResponse.BodyHandlers.ofString());
if (rejectedPart.statusCode() != 400 || !rejectedPart.body().contains("BadDigest"))
throw new AssertionError("Mismatched part Content-MD5 accepted");
var parts = client.send(signedUri(URI.create(base + "/objects/" + movie +
"?uploadId=" + upload + "&max-parts=1"), "GET", new byte[0], Map.of()),
HttpResponse.BodyHandlers.ofString());
@@ -202,9 +311,11 @@ public final class HttpTest {
HttpClient client = HttpClient.newHttpClient();
testObjects(client, base);
testListing(client, base);
testCopy(client, base);
testChecksums(client, base);
testMultipart(client, base);
testDelete(client, base);
System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE, MIME, ranges, listing, multipart");
System.out.println("HTTP tests passed: objects, copy, checksums, listing, multipart");
} finally {
server.stop(0);
executor.close();