Add ObjectStore source
This commit is contained in:
LunarSkyOSS committed
1 parent
8a3bee5dcc
commit
51e206f8df
14 files changed
+673
No files matched your search
@@ -0,0 +1,104 @@
|
||||
package cloud.lunarsky.store;
|
||||
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Comparator;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
public final class HttpTest {
|
||||
private static final String ACCESS = "TESTACCESSKEY123";
|
||||
private static final String SECRET = "test-secret-key-that-is-at-least-32-characters";
|
||||
private static final String REGION = "us-east-1";
|
||||
private static final DateTimeFormatter DATE =
|
||||
DateTimeFormatter.ofPattern("uuuuMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);
|
||||
|
||||
private static HttpRequest signed(String base, String method, String key, byte[] body) {
|
||||
URI uri = URI.create(base + "/objects/" + SigV4.encode(key, true));
|
||||
String host = uri.getAuthority();
|
||||
String date = DATE.format(Instant.now());
|
||||
String hash = SigV4.hex(SigV4.hash(body));
|
||||
String names = "host;x-amz-content-sha256;x-amz-date";
|
||||
String canonical = method + "\n" + uri.getRawPath() + "\n\n"
|
||||
+ "host:" + host + "\n"
|
||||
+ "x-amz-content-sha256:" + hash + "\n"
|
||||
+ "x-amz-date:" + date + "\n\n" + names + "\n" + hash;
|
||||
String scope = date.substring(0, 8) + "/" + REGION + "/s3/aws4_request";
|
||||
String toSign = "AWS4-HMAC-SHA256\n" + date + "\n" + scope + "\n"
|
||||
+ SigV4.hex(SigV4.hash(canonical.getBytes(StandardCharsets.UTF_8)));
|
||||
String signature = SigV4.hex(SigV4.hmac(
|
||||
SigV4.signingKey(SECRET, date.substring(0, 8), REGION), toSign));
|
||||
return HttpRequest.newBuilder(uri)
|
||||
.header("x-amz-date", date)
|
||||
.header("x-amz-content-sha256", hash)
|
||||
.header("authorization", "AWS4-HMAC-SHA256 Credential=" + ACCESS + "/"
|
||||
+ scope + ",SignedHeaders=" + names + ",Signature=" + signature)
|
||||
.method(method, body.length == 0
|
||||
? HttpRequest.BodyPublishers.noBody()
|
||||
: HttpRequest.BodyPublishers.ofByteArray(body))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static void status(int expected, HttpResponse<byte[]> response) {
|
||||
if (response.statusCode() != expected) {
|
||||
throw new AssertionError("Expected HTTP " + expected + ", got " + response.statusCode()
|
||||
+ ": " + new String(response.body(), StandardCharsets.UTF_8));
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Path root = Files.createTempDirectory("store-http-test-");
|
||||
var executor = Executors.newVirtualThreadPerTaskExecutor();
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16);
|
||||
DiskStore store = new DiskStore(root, 1024, 4096);
|
||||
try {
|
||||
var app = new Main(store,
|
||||
new SigV4(ACCESS, SECRET, REGION, Clock.systemUTC()), "objects");
|
||||
server.setExecutor(executor);
|
||||
server.createContext("/", app::handle);
|
||||
server.start();
|
||||
String base = "http://127.0.0.1:" + server.getAddress().getPort();
|
||||
HttpClient client = HttpClient.newHttpClient();
|
||||
status(200, client.send(HttpRequest.newBuilder(URI.create(base + "/health")).GET().build(),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
String key = "folder/moon-☾.txt";
|
||||
byte[] body = "independent storage test".getBytes(StandardCharsets.UTF_8);
|
||||
status(403, client.send(HttpRequest.newBuilder(URI.create(base + "/objects/" + key))
|
||||
.GET().build(), HttpResponse.BodyHandlers.ofByteArray()));
|
||||
status(200, client.send(signed(base, "PUT", key, body),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
var get = client.send(signed(base, "GET", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, get);
|
||||
if (!java.util.Arrays.equals(body, get.body())) throw new AssertionError("GET body mismatch");
|
||||
if (!"application/octet-stream".equals(get.headers().firstValue("content-type").orElse("")))
|
||||
throw new AssertionError("Unexpected content type");
|
||||
var head = client.send(signed(base, "HEAD", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray());
|
||||
status(200, head);
|
||||
if (head.body().length != 0) throw new AssertionError("HEAD returned a body");
|
||||
status(204, client.send(signed(base, "DELETE", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
status(404, client.send(signed(base, "GET", key, new byte[0]),
|
||||
HttpResponse.BodyHandlers.ofByteArray()));
|
||||
System.out.println("HTTP tests passed: health, authentication, PUT, GET, HEAD, DELETE");
|
||||
} finally {
|
||||
server.stop(0);
|
||||
executor.close();
|
||||
store.close();
|
||||
try (var paths = Files.walk(root)) {
|
||||
for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package cloud.lunarsky.store;
|
||||
import java.nio.file.*;
|
||||
import java.io.*;
|
||||
import java.util.Arrays;
|
||||
|
||||
public final class StoreTest {
|
||||
interface Operation {void run() throws Exception;}
|
||||
static void fails(int status,Operation operation)throws Exception{
|
||||
try{operation.run();throw new AssertionError("Expected "+status);}catch(StoreException error){if(error.status!=status)throw error;}
|
||||
}
|
||||
static DiskStore.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
|
||||
return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only);
|
||||
}
|
||||
public static void main(String[] args)throws Exception{
|
||||
var headers=new com.sun.net.httpserver.Headers();
|
||||
headers.set("host","examplebucket.s3.amazonaws.com");headers.set("range","bytes=0-9");
|
||||
headers.set("x-amz-date","20130524T000000Z");
|
||||
headers.set("x-amz-content-sha256","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||
headers.set("authorization","AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41");
|
||||
var clock=java.time.Clock.fixed(java.time.Instant.parse("2013-05-24T00:00:00Z"),java.time.ZoneOffset.UTC);
|
||||
var auth=new SigV4("AKIAIOSFODNN7EXAMPLE","wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","us-east-1",clock);
|
||||
var uri=java.net.URI.create("/test.txt");auth.verify("GET",uri,headers);
|
||||
fails(403,()->auth.verify("GET",java.net.URI.create("/other.txt"),headers));
|
||||
fails(403,()->auth.verify("DELETE",uri,headers));
|
||||
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",clock).verify("GET",uri,headers));
|
||||
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers));
|
||||
headers.add("host","duplicate");fails(403,()->auth.verify("GET",uri,headers));
|
||||
System.out.println("SigV4 official vector and tampering tests passed");
|
||||
Path root=Files.createTempDirectory("store-test-");
|
||||
try{
|
||||
try(var store=new DiskStore(root,8,10)){
|
||||
byte[] body={1,2,3,4,5,6};
|
||||
put(store,"../nested/☾",body,true);
|
||||
try(var obj=store.open("test","../nested/☾")){if(!Arrays.equals(body,obj.stream().readAllBytes()))throw new AssertionError("Roundtrip");}
|
||||
fails(412,()->put(store,"../nested/☾",new byte[]{9},true));
|
||||
fails(507,()->put(store,"second",body,true));
|
||||
fails(413,()->put(store,"large",new byte[9],true));
|
||||
fails(400,()->store.put("test","bad",new ByteArrayInputStream(body),6,"0".repeat(64),null,true));
|
||||
fails(400,()->store.put("test","short",new ByteArrayInputStream(body),7,SigV4.hex(SigV4.hash(body)),null,true));
|
||||
fails(404,()->store.open("test","bad"));
|
||||
put(store,"../nested/☾",new byte[]{9},false);
|
||||
put(store,"empty",new byte[0],true);
|
||||
try {
|
||||
new DiskStore(root,8,10);
|
||||
throw new AssertionError("A second process opened the same data directory");
|
||||
} catch(IOException expected) {
|
||||
if(!expected.getMessage().contains("already in use"))throw expected;
|
||||
}
|
||||
}
|
||||
try(var restarted=new DiskStore(root,8,10)){
|
||||
try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
|
||||
restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
|
||||
fails(404,()->restarted.open("test","../nested/☾"));
|
||||
}
|
||||
String id=SigV4.hex(SigV4.hash("test/empty".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
|
||||
Files.write(root.resolve("objects").resolve(id.substring(0,2)).resolve(id),new byte[]{1},StandardOpenOption.APPEND);
|
||||
try {
|
||||
new DiskStore(root,8,10);
|
||||
throw new AssertionError("A damaged object record was accepted");
|
||||
} catch(IOException expected) {
|
||||
if(!expected.getMessage().contains("object record"))throw expected;
|
||||
}
|
||||
try(var pending=Files.list(root.resolve("pending"))){if(pending.count()!=0)throw new AssertionError("Pending cleanup");}
|
||||
System.out.println("Java storage tests passed: roundtrip, quota, integrity, persistence, locking, corruption, delete");
|
||||
}finally{try(var paths=Files.walk(root)){for(var p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user