admin/ObjectStore
1
1 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor complex ObjectStore methods

Browse Source
This commit is contained in:
admin committed 2026-10-10 07:51:38 +02:00
1 parent b6b8e0adfb
commit 2276fee60c
9 files changed
+693 -500
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/cloud/lunarsky/store/StoreTest.java
+107 -79
View File
@@ -14,7 +14,7 @@ public final class StoreTest {
static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream");
}
public static void main(String[] args)throws Exception{
private static void testSignature() throws Exception {
var headers=new com.sun.net.httpserver.Headers();
headers.set("host","examplebucket.s3.amazonaws.com");headers.set("range","bytes=0-9");
headers.set("x-amz-date","20130524T000000Z");
@@ -29,88 +29,116 @@ public final class StoreTest {
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers));
headers.add("host","duplicate");fails(403,()->auth.verify("GET",uri,headers));
System.out.println("SigV4 official vector and tampering tests passed");
Path root=Files.createTempDirectory("store-test-");
try{
try(var store=new DiskStore(root,8,10)){
byte[] body={1,2,3,4,5,6};
put(store,"../nested/☾",body,true);
try(var obj=store.open("test","../nested/☾")){if(!Arrays.equals(body,obj.stream().readAllBytes()))throw new AssertionError("Roundtrip");}
fails(412,()->put(store,"../nested/☾",new byte[]{9},true));
fails(507,()->put(store,"second",body,true));
fails(413,()->put(store,"large",new byte[9],true));
fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true,"application/octet-stream"));
fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true,"application/octet-stream"));
fails(404,()->store.open("test","bad"));
put(store,"../nested/☾",new byte[]{9},false);
put(store,"empty",new byte[0],true);
if(store.list("test","","",100,null).objects().size()!=2)throw new AssertionError("List index");
if(store.objectCount()!=2||store.legacyObjects()!=0)throw new AssertionError("Object counts");
try {
new DiskStore(root,8,10);
throw new AssertionError("A second process opened the same data directory");
} catch(IOException expected) {
if(!expected.getMessage().contains("already in use"))throw expected;
}
}
try(var restarted=new DiskStore(root,8,10)){
try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
fails(404,()->restarted.open("test","../nested/☾"));
var uploads=new MultipartStore(restarted);
String upload=uploads.create("test","from-parts","text/plain");
byte[] part={1,2,3};
uploads.putPart(upload,"test","from-parts",1,new ByteArrayInputStream(part),part.length,
SigV4.hex(SigV4.hash(part)),null);
Files.writeString(root.resolve("pending-upload-id"),upload);
}
try(var resumed=new DiskStore(root,8,10)){
Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000");
Files.createDirectory(unfinished);
Files.write(unfinished.resolve("manifest"),new byte[]{1,2,3});
var uploads=new MultipartStore(resumed);
if(Files.exists(unfinished))throw new AssertionError("Unfinished multipart creation survived restart");
String upload=Files.readString(root.resolve("pending-upload-id"));
byte[] part={1,2,3};
String etag=SigV4.hex(MessageDigest.getInstance("MD5").digest(part));
uploads.complete(upload,"test","from-parts",List.of(new MultipartStorage.Part(1,etag)));
try(var obj=resumed.open("test","from-parts")){
if(!Arrays.equals(part,obj.stream().readAllBytes())||!obj.metadata().contentType().equals("text/plain"))
throw new AssertionError("Multipart restart");
}
resumed.delete("test","from-parts");
}
byte[] old={4,5,6};
String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId);
Files.createDirectories(oldPath.getParent());
ByteBuffer oldRecord=ByteBuffer.allocate(72+old.length).putLong(0x4c534f424a303031L)
.putLong(old.length).putLong(123456789L)
.put(MessageDigest.getInstance("MD5").digest(old)).put(SigV4.hash(old)).put(old);
Files.write(oldPath,oldRecord.array());
try(var migrated=new DiskStore(root,8,10)){
if(migrated.objectCount()!=2||migrated.legacyObjects()!=1)throw new AssertionError("Legacy counts");
try(var obj=migrated.open("test","legacy")){
if(!Arrays.equals(old,obj.stream().readAllBytes()))throw new AssertionError("Legacy read");
}
if(migrated.list("test","","",100,null).objects().stream().anyMatch(entry->entry.key().equals("legacy")))
throw new AssertionError("Legacy object appeared without a stored key");
put(migrated,"legacy",old,false);
if(migrated.objectCount()!=2||migrated.legacyObjects()!=0)throw new AssertionError("Legacy count after overwrite");
if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy")))
throw new AssertionError("Legacy overwrite was not indexed");
}
Files.delete(root.resolve("pending-upload-id"));
String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
}
private static void testInitialStore(Path root) throws Exception {
try(var store=new DiskStore(root,8,10)){
byte[] body={1,2,3,4,5,6};
put(store,"../nested/☾",body,true);
try(var obj=store.open("test","../nested/☾")){if(!Arrays.equals(body,obj.stream().readAllBytes()))throw new AssertionError("Roundtrip");}
fails(412,()->put(store,"../nested/☾",new byte[]{9},true));
fails(507,()->put(store,"second",body,true));
fails(413,()->put(store,"large",new byte[9],true));
fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true,"application/octet-stream"));
fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true,"application/octet-stream"));
fails(404,()->store.open("test","bad"));
put(store,"../nested/☾",new byte[]{9},false);
put(store,"empty",new byte[0],true);
if(store.list("test","","",100,null).objects().size()!=2)throw new AssertionError("List index");
if(store.objectCount()!=2||store.legacyObjects()!=0)throw new AssertionError("Object counts");
try {
new DiskStore(root,8,10);
throw new AssertionError("A damaged object record was accepted");
throw new AssertionError("A second process opened the same data directory");
} catch(IOException expected) {
if(!expected.getMessage().contains("object record"))throw expected;
if(!expected.getMessage().contains("already in use"))throw expected;
}
try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
}
}
private static void testRestart(Path root) throws Exception {
try(var restarted=new DiskStore(root,8,10)){
try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
fails(404,()->restarted.open("test","../nested/☾"));
var uploads=new MultipartStore(restarted);
String upload=uploads.create("test","from-parts","text/plain");
byte[] part={1,2,3};
uploads.putPart(upload,"test","from-parts",1,new ByteArrayInputStream(part),part.length,
SigV4.hex(SigV4.hash(part)),null);
Files.writeString(root.resolve("pending-upload-id"),upload);
}
}
private static void testMultipartRecovery(Path root) throws Exception {
try(var resumed=new DiskStore(root,8,10)){
Path unfinished=root.resolve("multipart/.creating-00000000-0000-0000-0000-000000000000");
Files.createDirectory(unfinished);
Files.write(unfinished.resolve("manifest"),new byte[]{1,2,3});
var uploads=new MultipartStore(resumed);
if(Files.exists(unfinished))throw new AssertionError("Unfinished multipart creation survived restart");
String upload=Files.readString(root.resolve("pending-upload-id"));
byte[] part={1,2,3};
String etag=SigV4.hex(MessageDigest.getInstance("MD5").digest(part));
uploads.complete(upload,"test","from-parts",List.of(new MultipartStorage.Part(1,etag)));
try(var obj=resumed.open("test","from-parts")){
if(!Arrays.equals(part,obj.stream().readAllBytes())||!obj.metadata().contentType().equals("text/plain"))
throw new AssertionError("Multipart restart");
}
resumed.delete("test","from-parts");
}
}
private static void testLegacyRecord(Path root) throws Exception {
byte[] old={4,5,6};
String oldId=SigV4.hex(SigV4.hash("test/legacy".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
Path oldPath=root.resolve("objects").resolve(oldId.substring(0,2)).resolve(oldId);
Files.createDirectories(oldPath.getParent());
ByteBuffer oldRecord=ByteBuffer.allocate(72+old.length).putLong(0x4c534f424a303031L)
.putLong(old.length).putLong(123456789L)
.put(MessageDigest.getInstance("MD5").digest(old)).put(SigV4.hash(old)).put(old);
Files.write(oldPath,oldRecord.array());
try(var migrated=new DiskStore(root,8,10)){
if(migrated.objectCount()!=2||migrated.legacyObjects()!=1)throw new AssertionError("Legacy counts");
try(var obj=migrated.open("test","legacy")){
if(!Arrays.equals(old,obj.stream().readAllBytes()))throw new AssertionError("Legacy read");
}
if(migrated.list("test","","",100,null).objects().stream().anyMatch(entry->entry.key().equals("legacy")))
throw new AssertionError("Legacy object appeared without a stored key");
put(migrated,"legacy",old,false);
if(migrated.objectCount()!=2||migrated.legacyObjects()!=0)throw new AssertionError("Legacy count after overwrite");
if(migrated.list("test","","",100,null).objects().stream().noneMatch(entry->entry.key().equals("legacy")))
throw new AssertionError("Legacy overwrite was not indexed");
}
}
private static void testCorruption(Path root) throws Exception {
Files.delete(root.resolve("pending-upload-id"));
String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
try {
new DiskStore(root,8,10);
throw new AssertionError("A damaged object record was accepted");
} catch(IOException expected) {
if(!expected.getMessage().contains("object record"))throw expected;
}
try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
}
public static void main(String[] args) throws Exception {
testSignature();
Path root = Files.createTempDirectory("store-test-");
try {
testInitialStore(root);
testRestart(root);
testMultipartRecovery(root);
testLegacyRecord(root);
testCorruption(root);
System.out.println("Java storage tests passed: roundtrip, quota, indexing, persistence, multipart recovery, legacy reads, locking, corruption, delete");
}finally{try(var paths=Files.walk(root)){for(var p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}}
} finally {
try (var paths = Files.walk(root)) {
for (var path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) Files.delete(path);
}
}
}
}