Refactor complex ObjectStore methods
This commit is contained in:
admin committed
1 parent
b6b8e0adfb
commit
2276fee60c
9 files changed
+693
-500
No files matched your search
@@ -66,6 +66,114 @@ public final class HttpTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static void testObjects(HttpClient client, String base) throws Exception {
|
||||
status(200, client.send(HttpRequest.newBuilder(URI.create(base + "/health")).GET().build(),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
String key = "folder/moon-☾.txt";
|
||||
byte[] body = "independent storage test".getBytes(StandardCharsets.UTF_8);
|
||||
status(403, client.send(HttpRequest.newBuilder(URI.create(base + "/objects/" + key))
|
||||
.GET().build(), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
status(200, client.send(signed(base, "PUT", key, body),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
String other = "folder/stars.txt";
|
||||
status(200, client.send(signedUri(URI.create(base + "/objects/" + other), "PUT",
|
||||
"stars".getBytes(StandardCharsets.UTF_8), Map.of("content-type", "text/plain")),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
var get = client.send(signed(base, "GET", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, get);
|
||||
if (!java.util.Arrays.equals(body, get.body())) throw new AssertionError("GET body mismatch");
|
||||
if (!"application/octet-stream".equals(get.headers().firstValue("content-type").orElse("")))
|
||||
throw new AssertionError("Unexpected content type");
|
||||
var typed = client.send(signed(base, "GET", other, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, typed);
|
||||
if (!"text/plain".equals(typed.headers().firstValue("content-type").orElse("")))
|
||||
throw new AssertionError("Stored content type missing");
|
||||
var partial = client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
|
||||
new byte[0], Map.of("range", "bytes=1-3")), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(206, partial);
|
||||
if (!"tar".equals(new String(partial.body(), StandardCharsets.UTF_8)) ||
|
||||
!"bytes 1-3/5".equals(partial.headers().firstValue("content-range").orElse("")))
|
||||
throw new AssertionError("Range response mismatch");
|
||||
status(416, client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
|
||||
new byte[0], Map.of("range", "bytes=20-30")), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
}
|
||||
|
||||
private static void testListing(HttpClient client, String base) throws Exception {
|
||||
var listed = client.send(signedUri(URI.create(base + "/objects?list-type=2&prefix=folder%2F"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (listed.statusCode() != 200 || !listed.body().contains("<Key>folder/stars.txt</Key>") ||
|
||||
!listed.body().contains("<Key>folder/moon-☾.txt</Key>"))
|
||||
throw new AssertionError("ListObjectsV2 failed: " + listed.body());
|
||||
var page = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=1"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (page.statusCode() != 200 || !page.body().contains("<IsTruncated>true</IsTruncated>"))
|
||||
throw new AssertionError("List pagination failed: " + page.body());
|
||||
String token = page.body().split("<NextContinuationToken>")[1].split("</NextContinuationToken>")[0];
|
||||
var next = client.send(signedUri(URI.create(base + "/objects?list-type=2&continuation-token=" + token),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (next.statusCode() != 200 || !next.body().contains("<Key>folder/stars.txt</Key>"))
|
||||
throw new AssertionError("List continuation failed: " + next.body());
|
||||
var grouped = client.send(signedUri(URI.create(base + "/objects?list-type=2&delimiter=%2F"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (grouped.statusCode() != 200 || !grouped.body().contains("<CommonPrefixes><Prefix>folder/</Prefix></CommonPrefixes>") ||
|
||||
grouped.body().contains("<Contents>"))
|
||||
throw new AssertionError("Delimiter listing failed: " + grouped.body());
|
||||
var encoded = client.send(signedUri(URI.create(base + "/objects?list-type=2&encoding-type=url"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (encoded.statusCode() != 200 || !encoded.body().contains("folder%2Fmoon-%E2%98%BE.txt"))
|
||||
throw new AssertionError("Encoded listing failed: " + encoded.body());
|
||||
var emptyPage = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=0"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (emptyPage.statusCode() != 200 || !emptyPage.body().contains("<KeyCount>0</KeyCount>"))
|
||||
throw new AssertionError("Empty list page failed: " + emptyPage.body());
|
||||
}
|
||||
|
||||
private static void testMultipart(HttpClient client, String base) throws Exception {
|
||||
String movie = "folder/video.mp4";
|
||||
URI initiate = URI.create(base + "/objects/" + movie + "?uploads=");
|
||||
var created = client.send(signedUri(initiate, "POST", new byte[0],
|
||||
Map.of("content-type", "video/mp4")), HttpResponse.BodyHandlers.ofString());
|
||||
if (created.statusCode() != 200) throw new AssertionError("Multipart initiation failed: " + created.body());
|
||||
String upload = created.body().split("<UploadId>")[1].split("</UploadId>")[0];
|
||||
byte[] first = "hello ".getBytes(StandardCharsets.UTF_8);
|
||||
byte[] second = "world".getBytes(StandardCharsets.UTF_8);
|
||||
var partOne = client.send(signedUri(URI.create(base + "/objects/" + movie +
|
||||
"?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
||||
var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie +
|
||||
"?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, partOne); status(200, partTwo);
|
||||
String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
|
||||
partOne.headers().firstValue("etag").orElseThrow() +
|
||||
"</ETag></Part><Part><PartNumber>2</PartNumber><ETag>" +
|
||||
partTwo.headers().firstValue("etag").orElseThrow() +
|
||||
"</ETag></Part></CompleteMultipartUpload>";
|
||||
status(200, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload),
|
||||
"POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
var assembled = client.send(signed(base, "GET", movie, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, assembled);
|
||||
if (!"hello world".equals(new String(assembled.body(), StandardCharsets.UTF_8)) ||
|
||||
!"video/mp4".equals(assembled.headers().firstValue("content-type").orElse("")))
|
||||
throw new AssertionError("Completed multipart object mismatch");
|
||||
var abandoned = client.send(signedUri(URI.create(base + "/objects/abandoned?uploads="),
|
||||
"POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
String abandonedId = abandoned.body().split("<UploadId>")[1].split("</UploadId>")[0];
|
||||
status(204, client.send(signedUri(URI.create(base + "/objects/abandoned?uploadId=" + abandonedId),
|
||||
"DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
}
|
||||
|
||||
private static void testDelete(HttpClient client, String base) throws Exception {
|
||||
String key = "folder/moon-☾.txt";
|
||||
var head = client.send(signed(base, "HEAD", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, head);
|
||||
if (head.body().length != 0) throw new AssertionError("HEAD returned a body");
|
||||
status(204, client.send(signed(base, "DELETE", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
status(404, client.send(signed(base, "GET", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Path root = Files.createTempDirectory("store-http-test-");
|
||||
var executor = Executors.newVirtualThreadPerTaskExecutor();
|
||||
@@ -79,101 +187,10 @@ public final class HttpTest {
|
||||
server.start();
|
||||
String base = "http://127.0.0.1:" + server.getAddress().getPort();
|
||||
HttpClient client = HttpClient.newHttpClient();
|
||||
status(200, client.send(HttpRequest.newBuilder(URI.create(base + "/health")).GET().build(),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
String key = "folder/moon-☾.txt";
|
||||
byte[] body = "independent storage test".getBytes(StandardCharsets.UTF_8);
|
||||
status(403, client.send(HttpRequest.newBuilder(URI.create(base + "/objects/" + key))
|
||||
.GET().build(), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
status(200, client.send(signed(base, "PUT", key, body),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
String other = "folder/stars.txt";
|
||||
status(200, client.send(signedUri(URI.create(base + "/objects/" + other), "PUT",
|
||||
"stars".getBytes(StandardCharsets.UTF_8), Map.of("content-type", "text/plain")),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
var get = client.send(signed(base, "GET", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, get);
|
||||
if (!java.util.Arrays.equals(body, get.body())) throw new AssertionError("GET body mismatch");
|
||||
if (!"application/octet-stream".equals(get.headers().firstValue("content-type").orElse("")))
|
||||
throw new AssertionError("Unexpected content type");
|
||||
var typed = client.send(signed(base, "GET", other, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, typed);
|
||||
if (!"text/plain".equals(typed.headers().firstValue("content-type").orElse("")))
|
||||
throw new AssertionError("Stored content type missing");
|
||||
var partial = client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
|
||||
new byte[0], Map.of("range", "bytes=1-3")), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(206, partial);
|
||||
if (!"tar".equals(new String(partial.body(), StandardCharsets.UTF_8)) ||
|
||||
!"bytes 1-3/5".equals(partial.headers().firstValue("content-range").orElse("")))
|
||||
throw new AssertionError("Range response mismatch");
|
||||
status(416, client.send(signedUri(URI.create(base + "/objects/" + other), "GET",
|
||||
new byte[0], Map.of("range", "bytes=20-30")), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
var listed = client.send(signedUri(URI.create(base + "/objects?list-type=2&prefix=folder%2F"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (listed.statusCode() != 200 || !listed.body().contains("<Key>folder/stars.txt</Key>") ||
|
||||
!listed.body().contains("<Key>folder/moon-☾.txt</Key>"))
|
||||
throw new AssertionError("ListObjectsV2 failed: " + listed.body());
|
||||
var page = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=1"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (page.statusCode() != 200 || !page.body().contains("<IsTruncated>true</IsTruncated>"))
|
||||
throw new AssertionError("List pagination failed: " + page.body());
|
||||
String token = page.body().split("<NextContinuationToken>")[1].split("</NextContinuationToken>")[0];
|
||||
var next = client.send(signedUri(URI.create(base + "/objects?list-type=2&continuation-token=" + token),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (next.statusCode() != 200 || !next.body().contains("<Key>folder/stars.txt</Key>"))
|
||||
throw new AssertionError("List continuation failed: " + next.body());
|
||||
var grouped = client.send(signedUri(URI.create(base + "/objects?list-type=2&delimiter=%2F"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (grouped.statusCode() != 200 || !grouped.body().contains("<CommonPrefixes><Prefix>folder/</Prefix></CommonPrefixes>") ||
|
||||
grouped.body().contains("<Contents>"))
|
||||
throw new AssertionError("Delimiter listing failed: " + grouped.body());
|
||||
var encoded = client.send(signedUri(URI.create(base + "/objects?list-type=2&encoding-type=url"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (encoded.statusCode() != 200 || !encoded.body().contains("folder%2Fmoon-%E2%98%BE.txt"))
|
||||
throw new AssertionError("Encoded listing failed: " + encoded.body());
|
||||
var emptyPage = client.send(signedUri(URI.create(base + "/objects?list-type=2&max-keys=0"),
|
||||
"GET", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
if (emptyPage.statusCode() != 200 || !emptyPage.body().contains("<KeyCount>0</KeyCount>"))
|
||||
throw new AssertionError("Empty list page failed: " + emptyPage.body());
|
||||
String movie = "folder/video.mp4";
|
||||
URI initiate = URI.create(base + "/objects/" + movie + "?uploads=");
|
||||
var created = client.send(signedUri(initiate, "POST", new byte[0],
|
||||
Map.of("content-type", "video/mp4")), HttpResponse.BodyHandlers.ofString());
|
||||
if (created.statusCode() != 200) throw new AssertionError("Multipart initiation failed: " + created.body());
|
||||
String upload = created.body().split("<UploadId>")[1].split("</UploadId>")[0];
|
||||
byte[] first = "hello ".getBytes(StandardCharsets.UTF_8);
|
||||
byte[] second = "world".getBytes(StandardCharsets.UTF_8);
|
||||
var partOne = client.send(signedUri(URI.create(base + "/objects/" + movie +
|
||||
"?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
||||
var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie +
|
||||
"?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, partOne); status(200, partTwo);
|
||||
String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
|
||||
partOne.headers().firstValue("etag").orElseThrow() +
|
||||
"</ETag></Part><Part><PartNumber>2</PartNumber><ETag>" +
|
||||
partTwo.headers().firstValue("etag").orElseThrow() +
|
||||
"</ETag></Part></CompleteMultipartUpload>";
|
||||
status(200, client.send(signedUri(URI.create(base + "/objects/" + movie + "?uploadId=" + upload),
|
||||
"POST", completion.getBytes(StandardCharsets.UTF_8), Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
var assembled = client.send(signed(base, "GET", movie, new byte[0]), HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, assembled);
|
||||
if (!"hello world".equals(new String(assembled.body(), StandardCharsets.UTF_8)) ||
|
||||
!"video/mp4".equals(assembled.headers().firstValue("content-type").orElse("")))
|
||||
throw new AssertionError("Completed multipart object mismatch");
|
||||
var abandoned = client.send(signedUri(URI.create(base + "/objects/abandoned?uploads="),
|
||||
"POST", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofString());
|
||||
String abandonedId = abandoned.body().split("<UploadId>")[1].split("</UploadId>")[0];
|
||||
status(204, client.send(signedUri(URI.create(base + "/objects/abandoned?uploadId=" + abandonedId),
|
||||
"DELETE", new byte[0], Map.of()), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
var head = client.send(signed(base, "HEAD", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, head);
|
||||
if (head.body().length != 0) throw new AssertionError("HEAD returned a body");
|
||||
status(204, client.send(signed(base, "DELETE", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
status(404, client.send(signed(base, "GET", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
testObjects(client, base);
|
||||
testListing(client, base);
|
||||
testMultipart(client, base);
|
||||
testDelete(client, base);
|
||||
System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE, MIME, ranges, listing, multipart");
|
||||
} finally {
|
||||
server.stop(0);
|
||||
|
||||
@@ -14,7 +14,7 @@ public final class StoreTest {
|
||||
static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
|
||||
return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream");
|
||||
}
|
||||
public static void main(String[] args)throws Exception{
|
||||
private static void testSignature() throws Exception {
|
||||
var headers=new com.sun.net.httpserver.Headers();
|
||||
headers.set("host","examplebucket.s3.amazonaws.com");headers.set("range","bytes=0-9");
|
||||
headers.set("x-amz-date","20130524T000000Z");
|
||||
@@ -29,88 +29,116 @@ public final class StoreTest {
|
||||
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers));
|
||||
headers.add("host","duplicate");fails(403,()->auth.verify("GET",uri,headers));
|
||||
System.out.println("SigV4 official vector and tampering tests passed");
|
||||
Path root=Files.createTempDirectory("store-test-");
|
||||
try{
|
||||
try(var store=new DiskStore(root,8,10)){
|
||||
byte[] body={1,2,3,4,5,6};
|
||||
put(store,"../nested/☾",body,true);
|
||||
try(var obj=store.open("test","../nested/☾")){if(!Arrays.equals(body,obj.stream().readAllBytes()))throw new AssertionError("Roundtrip");}
|
||||
fails(412,()->put(store,"../nested/☾",new byte[]{9},true));
|
||||
fails(507,()->put(store,"second",body,true));
|
||||
fails(413,()->put(store,"large",new byte[9],true));
|
||||
fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true,"application/octet-stream"));
|
||||
fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true,"application/octet-stream"));
|
||||
fails(404,()->store.open("test","bad"));
|
||||
put(store,"../nested/☾",new byte[]{9},false);
|
||||
put(store,"empty",new byte[0],true);
|
||||
if(store.list("test","","",100,null).objects().size()!=2)throw new AssertionError("List index");
|
||||
if(store.objectCount()!=2||store.legacyObjects()!=0)throw new AssertionError("Object counts");
|
||||
try {
|
||||
new DiskStore(root,8,10);
|
||||
throw new AssertionError("A second process opened the same data directory");
|
||||
} catch(IOException expected) {
|
||||
if(!expected.getMessage().contains("already in use"))throw expected;
|
||||
}
|
||||
}
|
||||
try(var restarted=new DiskStore(root,8,10)){
|
||||
try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
|
||||
if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
|
||||
restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
|
||||
fails(404,()->restarted.open("test","../nested/☾"));
|
||||
var uploads=new MultipartStore(restarted);
|
||||
String upload=uploads.create("test","from-parts","text/plain");
|
||||
byte[] part={1,2,3};
|
||||
uploads.putPart(upload,"test","from-parts",1,new ByteArrayInputStream(part),part.length,
|
||||
SigV4.hex(SigV4.hash(part)),null);
|
||||
Files.writeString(root.resolve("pending-upload-id"),upload);
|
||||
}
|
||||
try(var resumed=new DiskStore(root,8,10)){
|
||||
Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000");
|
||||
Files.createDirectory(unfinished);
|
||||
Files.write(unfinished.resolve("manifest"),new byte[]{1,2,3});
|
||||
var uploads=new MultipartStore(resumed);
|
||||
if(Files.exists(unfinished))throw new AssertionError("Unfinished multipart creation survived restart");
|
||||
String upload=Files.readString(root.resolve("pending-upload-id"));
|
||||
byte[] part={1,2,3};
|
||||
String etag=SigV4.hex(MessageDigest.getInstance("MD5").digest(part));
|
||||
uploads.complete(upload,"test","from-parts",List.of(new MultipartStorage.Part(1,etag)));
|
||||
try(var obj=resumed.open("test","from-parts")){
|
||||
if(!Arrays.equals(part,obj.stream().readAllBytes())||!obj.metadata().contentType().equals("text/plain"))
|
||||
throw new AssertionError("Multipart restart");
|
||||
}
|
||||
resumed.delete("test","from-parts");
|
||||
}
|
||||
byte[] old={4,5,6};
|
||||
String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
|
||||
Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId);
|
||||
Files.createDirectories(oldPath.getParent());
|
||||
ByteBuffer oldRecord=ByteBuffer.allocate(72+old.length).putLong(0x4c534f424a303031L)
|
||||
.putLong(old.length).putLong(123456789L)
|
||||
.put(MessageDigest.getInstance("MD5").digest(old)).put(SigV4.hash(old)).put(old);
|
||||
Files.write(oldPath,oldRecord.array());
|
||||
try(var migrated=new DiskStore(root,8,10)){
|
||||
if(migrated.objectCount()!=2||migrated.legacyObjects()!=1)throw new AssertionError("Legacy counts");
|
||||
try(var obj=migrated.open("test","legacy")){
|
||||
if(!Arrays.equals(old,obj.stream().readAllBytes()))throw new AssertionError("Legacy read");
|
||||
}
|
||||
if(migrated.list("test","","",100,null).objects().stream().anyMatch(entry->entry.key().equals("legacy")))
|
||||
throw new AssertionError("Legacy object appeared without a stored key");
|
||||
put(migrated,"legacy",old,false);
|
||||
if(migrated.objectCount()!=2||migrated.legacyObjects()!=0)throw new AssertionError("Legacy count after overwrite");
|
||||
if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy")))
|
||||
throw new AssertionError("Legacy overwrite was not indexed");
|
||||
}
|
||||
Files.delete(root.resolve("pending-upload-id"));
|
||||
String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
|
||||
Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
|
||||
}
|
||||
|
||||
private static void testInitialStore(Path root) throws Exception {
|
||||
try(var store=new DiskStore(root,8,10)){
|
||||
byte[] body={1,2,3,4,5,6};
|
||||
put(store,"../nested/☾",body,true);
|
||||
try(var obj=store.open("test","../nested/☾")){if(!Arrays.equals(body,obj.stream().readAllBytes()))throw new AssertionError("Roundtrip");}
|
||||
fails(412,()->put(store,"../nested/☾",new byte[]{9},true));
|
||||
fails(507,()->put(store,"second",body,true));
|
||||
fails(413,()->put(store,"large",new byte[9],true));
|
||||
fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true,"application/octet-stream"));
|
||||
fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true,"application/octet-stream"));
|
||||
fails(404,()->store.open("test","bad"));
|
||||
put(store,"../nested/☾",new byte[]{9},false);
|
||||
put(store,"empty",new byte[0],true);
|
||||
if(store.list("test","","",100,null).objects().size()!=2)throw new AssertionError("List index");
|
||||
if(store.objectCount()!=2||store.legacyObjects()!=0)throw new AssertionError("Object counts");
|
||||
try {
|
||||
new DiskStore(root,8,10);
|
||||
throw new AssertionError("A damaged object record was accepted");
|
||||
throw new AssertionError("A second process opened the same data directory");
|
||||
} catch(IOException expected) {
|
||||
if(!expected.getMessage().contains("object record"))throw expected;
|
||||
if(!expected.getMessage().contains("already in use"))throw expected;
|
||||
}
|
||||
try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
|
||||
}
|
||||
}
|
||||
|
||||
private static void testRestart(Path root) throws Exception {
|
||||
try(var restarted=new DiskStore(root,8,10)){
|
||||
try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
|
||||
if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
|
||||
restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
|
||||
fails(404,()->restarted.open("test","../nested/☾"));
|
||||
var uploads=new MultipartStore(restarted);
|
||||
String upload=uploads.create("test","from-parts","text/plain");
|
||||
byte[] part={1,2,3};
|
||||
uploads.putPart(upload,"test","from-parts",1,new ByteArrayInputStream(part),part.length,
|
||||
SigV4.hex(SigV4.hash(part)),null);
|
||||
Files.writeString(root.resolve("pending-upload-id"),upload);
|
||||
}
|
||||
}
|
||||
|
||||
private static void testMultipartRecovery(Path root) throws Exception {
|
||||
try(var resumed=new DiskStore(root,8,10)){
|
||||
Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000");
|
||||
Files.createDirectory(unfinished);
|
||||
Files.write(unfinished.resolve("manifest"),new byte[]{1,2,3});
|
||||
var uploads=new MultipartStore(resumed);
|
||||
if(Files.exists(unfinished))throw new AssertionError("Unfinished multipart creation survived restart");
|
||||
String upload=Files.readString(root.resolve("pending-upload-id"));
|
||||
byte[] part={1,2,3};
|
||||
String etag=SigV4.hex(MessageDigest.getInstance("MD5").digest(part));
|
||||
uploads.complete(upload,"test","from-parts",List.of(new MultipartStorage.Part(1,etag)));
|
||||
try(var obj=resumed.open("test","from-parts")){
|
||||
if(!Arrays.equals(part,obj.stream().readAllBytes())||!obj.metadata().contentType().equals("text/plain"))
|
||||
throw new AssertionError("Multipart restart");
|
||||
}
|
||||
resumed.delete("test","from-parts");
|
||||
}
|
||||
}
|
||||
|
||||
private static void testLegacyRecord(Path root) throws Exception {
|
||||
byte[] old={4,5,6};
|
||||
String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
|
||||
Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId);
|
||||
Files.createDirectories(oldPath.getParent());
|
||||
ByteBuffer oldRecord=ByteBuffer.allocate(72+old.length).putLong(0x4c534f424a303031L)
|
||||
.putLong(old.length).putLong(123456789L)
|
||||
.put(MessageDigest.getInstance("MD5").digest(old)).put(SigV4.hash(old)).put(old);
|
||||
Files.write(oldPath,oldRecord.array());
|
||||
try(var migrated=new DiskStore(root,8,10)){
|
||||
if(migrated.objectCount()!=2||migrated.legacyObjects()!=1)throw new AssertionError("Legacy counts");
|
||||
try(var obj=migrated.open("test","legacy")){
|
||||
if(!Arrays.equals(old,obj.stream().readAllBytes()))throw new AssertionError("Legacy read");
|
||||
}
|
||||
if(migrated.list("test","","",100,null).objects().stream().anyMatch(entry->entry.key().equals("legacy")))
|
||||
throw new AssertionError("Legacy object appeared without a stored key");
|
||||
put(migrated,"legacy",old,false);
|
||||
if(migrated.objectCount()!=2||migrated.legacyObjects()!=0)throw new AssertionError("Legacy count after overwrite");
|
||||
if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy")))
|
||||
throw new AssertionError("Legacy overwrite was not indexed");
|
||||
}
|
||||
}
|
||||
|
||||
private static void testCorruption(Path root) throws Exception {
|
||||
Files.delete(root.resolve("pending-upload-id"));
|
||||
String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
|
||||
Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
|
||||
try {
|
||||
new DiskStore(root,8,10);
|
||||
throw new AssertionError("A damaged object record was accepted");
|
||||
} catch(IOException expected) {
|
||||
if(!expected.getMessage().contains("object record"))throw expected;
|
||||
}
|
||||
try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
testSignature();
|
||||
Path root = Files.createTempDirectory("store-test-");
|
||||
try {
|
||||
testInitialStore(root);
|
||||
testRestart(root);
|
||||
testMultipartRecovery(root);
|
||||
testLegacyRecord(root);
|
||||
testCorruption(root);
|
||||
System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete");
|
||||
}finally{try(var paths=Files.walk(root)){for(var p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}}
|
||||
} finally {
|
||||
try (var paths = Files.walk(root)) {
|
||||
for (var path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) Files.delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user