Put Java statements on separate lines
This commit is contained in:
1 parent
01c82a8223
commit
11bfe71124
15 files changed
+228
-85
No files matched your search
@@ -109,7 +109,9 @@ public final class Cli {
|
|||||||
}
|
}
|
||||||
if (verify) {
|
if (verify) {
|
||||||
MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
|
MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
|
||||||
byte[] buffer = new byte[65536]; long count = 0; int n;
|
byte[] buffer = new byte[65536];
|
||||||
|
long count = 0;
|
||||||
|
int n;
|
||||||
while ((n = input.read(buffer)) != -1) {
|
while ((n = input.read(buffer)) != -1) {
|
||||||
count += n;
|
count += n;
|
||||||
sha.update(buffer, 0, n);
|
sha.update(buffer, 0, n);
|
||||||
|
|||||||
@@ -148,7 +148,8 @@ public final class ClusterMigrate {
|
|||||||
}
|
}
|
||||||
connection.commit();
|
connection.commit();
|
||||||
} catch (SQLException | IOException | RuntimeException error) {
|
} catch (SQLException | IOException | RuntimeException error) {
|
||||||
try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
|
try { connection.rollback(); }
|
||||||
|
catch (SQLException rollback) { error.addSuppressed(rollback); }
|
||||||
if (error instanceof IOException io) throw io;
|
if (error instanceof IOException io) throw io;
|
||||||
if (error instanceof SQLException sql) throw sql;
|
if (error instanceof SQLException sql) throw sql;
|
||||||
throw (RuntimeException) error;
|
throw (RuntimeException) error;
|
||||||
|
|||||||
@@ -252,7 +252,8 @@ public final class ClusterNode implements AutoCloseable {
|
|||||||
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
|
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
|
||||||
server.stop(5);
|
server.stop(5);
|
||||||
executor.close();
|
executor.close();
|
||||||
try { node.close(); } catch (IOException error) { System.err.println("Node close failed: " + error); }
|
try { node.close(); }
|
||||||
|
catch (IOException error) { System.err.println("Node close failed: " + error); }
|
||||||
}));
|
}));
|
||||||
server.start();
|
server.start();
|
||||||
System.out.println("ObjectStore cluster node listening on :" + port);
|
System.out.println("ObjectStore cluster node listening on :" + port);
|
||||||
|
|||||||
@@ -38,8 +38,12 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
boolean testNodeDomains) throws IOException {
|
boolean testNodeDomains) throws IOException {
|
||||||
if (jdbcUrl == null || !jdbcUrl.startsWith("jdbc:postgresql://") || user == null || password == null)
|
if (jdbcUrl == null || !jdbcUrl.startsWith("jdbc:postgresql://") || user == null || password == null)
|
||||||
throw new IllegalArgumentException("Invalid metadata database configuration");
|
throw new IllegalArgumentException("Invalid metadata database configuration");
|
||||||
this.jdbcUrl = jdbcUrl; this.user = user; this.password = password;
|
this.jdbcUrl = jdbcUrl;
|
||||||
this.configuredBucket = bucket; this.maxObject = maxObject; this.maxTotal = maxTotal;
|
this.user = user;
|
||||||
|
this.password = password;
|
||||||
|
this.configuredBucket = bucket;
|
||||||
|
this.maxObject = maxObject;
|
||||||
|
this.maxTotal = maxTotal;
|
||||||
this.testNodeDomains = testNodeDomains;
|
this.testNodeDomains = testNodeDomains;
|
||||||
try (Connection connection = connect()) {
|
try (Connection connection = connect()) {
|
||||||
int format = SchemaMigrator.prepare(connection, bucket);
|
int format = SchemaMigrator.prepare(connection, bucket);
|
||||||
@@ -88,7 +92,8 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
int count = input.read(buffer, 0, (int) Math.min(buffer.length, remaining));
|
int count = input.read(buffer, 0, (int) Math.min(buffer.length, remaining));
|
||||||
if (count < 0) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length");
|
if (count < 0) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length");
|
||||||
if (count == 0) continue;
|
if (count == 0) continue;
|
||||||
sha.update(buffer, 0, count); md5.update(buffer, 0, count);
|
sha.update(buffer, 0, count);
|
||||||
|
md5.update(buffer, 0, count);
|
||||||
output.write(buffer, 0, count);
|
output.write(buffer, 0, count);
|
||||||
remaining -= count;
|
remaining -= count;
|
||||||
}
|
}
|
||||||
@@ -169,8 +174,11 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
"INSERT INTO cluster_segments (generation, ordinal, segment_id, length, sha256, replicas, replica_ids) VALUES (?, ?, ?, ?, ?, 'v2', ?)")) {
|
"INSERT INTO cluster_segments (generation, ordinal, segment_id, length, sha256, replicas, replica_ids) VALUES (?, ?, ?, ?, ?, 'v2', ?)")) {
|
||||||
for (int i = 0; i < segments.size(); i++) {
|
for (int i = 0; i < segments.size(); i++) {
|
||||||
Segment segment = segments.get(i);
|
Segment segment = segments.get(i);
|
||||||
insert.setObject(1, generation); insert.setInt(2, i); insert.setObject(3, segment.id());
|
insert.setObject(1, generation);
|
||||||
insert.setInt(4, segment.length()); insert.setBytes(5, segment.hash());
|
insert.setInt(2, i);
|
||||||
|
insert.setObject(3, segment.id());
|
||||||
|
insert.setInt(4, segment.length());
|
||||||
|
insert.setBytes(5, segment.hash());
|
||||||
insert.setArray(6, connection.createArrayOf("uuid", segment.replicas().toArray()));
|
insert.setArray(6, connection.createArrayOf("uuid", segment.replicas().toArray()));
|
||||||
insert.addBatch();
|
insert.addBatch();
|
||||||
}
|
}
|
||||||
@@ -178,13 +186,18 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
}
|
}
|
||||||
try (PreparedStatement update = connection.prepareStatement(
|
try (PreparedStatement update = connection.prepareStatement(
|
||||||
"INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, length=EXCLUDED.length, modified=EXCLUDED.modified, etag=EXCLUDED.etag, sha256=EXCLUDED.sha256, content_type=EXCLUDED.content_type")) {
|
"INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, length=EXCLUDED.length, modified=EXCLUDED.modified, etag=EXCLUDED.etag, sha256=EXCLUDED.sha256, content_type=EXCLUDED.content_type")) {
|
||||||
bindObject(update, metadata, generation); update.executeUpdate();
|
bindObject(update, metadata, generation);
|
||||||
|
update.executeUpdate();
|
||||||
}
|
}
|
||||||
try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) {
|
try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) {
|
||||||
update.setLong(1, used - Math.max(0, previous) + length); update.setString(2, bucket); update.executeUpdate();
|
update.setLong(1, used - Math.max(0, previous) + length);
|
||||||
|
update.setString(2, bucket);
|
||||||
|
update.executeUpdate();
|
||||||
}
|
}
|
||||||
try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) {
|
try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_tombstones WHERE bucket=? AND object_key=?")) {
|
||||||
delete.setString(1, bucket); delete.setString(2, key); delete.executeUpdate();
|
delete.setString(1, bucket);
|
||||||
|
delete.setString(2, key);
|
||||||
|
delete.executeUpdate();
|
||||||
}
|
}
|
||||||
connection.commit();
|
connection.commit();
|
||||||
} catch (SQLException | RuntimeException error) {
|
} catch (SQLException | RuntimeException error) {
|
||||||
@@ -204,7 +217,8 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
UUID generation;
|
UUID generation;
|
||||||
try (PreparedStatement query = connection.prepareStatement(
|
try (PreparedStatement query = connection.prepareStatement(
|
||||||
"SELECT generation, length, modified, etag, sha256, content_type FROM cluster_objects WHERE bucket=? AND object_key=?")) {
|
"SELECT generation, length, modified, etag, sha256, content_type FROM cluster_objects WHERE bucket=? AND object_key=?")) {
|
||||||
query.setString(1, bucket); query.setString(2, key);
|
query.setString(1, bucket);
|
||||||
|
query.setString(2, key);
|
||||||
try (ResultSet result = query.executeQuery()) {
|
try (ResultSet result = query.executeQuery()) {
|
||||||
if (!result.next()) throw new StoreException(404, "NoSuchKey", "Object not found");
|
if (!result.next()) throw new StoreException(404, "NoSuchKey", "Object not found");
|
||||||
generation = (UUID) result.getObject(1);
|
generation = (UUID) result.getObject(1);
|
||||||
@@ -245,17 +259,23 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
long used = lockUsage(connection, bucket);
|
long used = lockUsage(connection, bucket);
|
||||||
long previous = currentLength(connection, bucket, key);
|
long previous = currentLength(connection, bucket, key);
|
||||||
try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_objects WHERE bucket=? AND object_key=?")) {
|
try (PreparedStatement delete = connection.prepareStatement("DELETE FROM cluster_objects WHERE bucket=? AND object_key=?")) {
|
||||||
delete.setString(1, bucket); delete.setString(2, key); delete.executeUpdate();
|
delete.setString(1, bucket);
|
||||||
|
delete.setString(2, key);
|
||||||
|
delete.executeUpdate();
|
||||||
}
|
}
|
||||||
try (PreparedStatement update = connection.prepareStatement(
|
try (PreparedStatement update = connection.prepareStatement(
|
||||||
"INSERT INTO cluster_tombstones VALUES (?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, deleted_at=EXCLUDED.deleted_at")) {
|
"INSERT INTO cluster_tombstones VALUES (?, ?, ?, ?) ON CONFLICT (bucket, object_key) DO UPDATE SET generation=EXCLUDED.generation, deleted_at=EXCLUDED.deleted_at")) {
|
||||||
update.setString(1, bucket); update.setString(2, key);
|
update.setString(1, bucket);
|
||||||
update.setObject(3, UUID.randomUUID()); update.setLong(4, Instant.now().toEpochMilli());
|
update.setString(2, key);
|
||||||
|
update.setObject(3, UUID.randomUUID());
|
||||||
|
update.setLong(4, Instant.now().toEpochMilli());
|
||||||
update.executeUpdate();
|
update.executeUpdate();
|
||||||
}
|
}
|
||||||
if (previous >= 0) {
|
if (previous >= 0) {
|
||||||
try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) {
|
try (PreparedStatement update = connection.prepareStatement("UPDATE cluster_usage SET used_bytes=? WHERE bucket=?")) {
|
||||||
update.setLong(1, used - previous); update.setString(2, bucket); update.executeUpdate();
|
update.setLong(1, used - previous);
|
||||||
|
update.setString(2, bucket);
|
||||||
|
update.executeUpdate();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
connection.commit();
|
connection.commit();
|
||||||
@@ -297,10 +317,18 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
if (!key.startsWith(prefix)) break;
|
if (!key.startsWith(prefix)) break;
|
||||||
if (after != null && key.compareTo(after) <= 0) continue;
|
if (after != null && key.compareTo(after) <= 0) continue;
|
||||||
String group = commonPrefix(key, prefix, delimiter);
|
String group = commonPrefix(key, prefix, delimiter);
|
||||||
if (group != null && group.equals(activePrefix)) { lastKey = key; continue; }
|
if (group != null && group.equals(activePrefix)) {
|
||||||
if (entries.size() + prefixes.size() >= maxKeys) { truncated = true; break; }
|
lastKey = key;
|
||||||
if (group != null) { prefixes.add(group); activePrefix = group; }
|
continue;
|
||||||
else {
|
}
|
||||||
|
if (entries.size() + prefixes.size() >= maxKeys) {
|
||||||
|
truncated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (group != null) {
|
||||||
|
prefixes.add(group);
|
||||||
|
activePrefix = group;
|
||||||
|
} else {
|
||||||
entries.add(new ListedObject(key, new Metadata(result.getLong(2), result.getLong(3),
|
entries.add(new ListedObject(key, new Metadata(result.getLong(2), result.getLong(3),
|
||||||
result.getString(4), result.getBytes(5), bucket, key, result.getString(6))));
|
result.getString(4), result.getBytes(5), bucket, key, result.getString(6))));
|
||||||
activePrefix = null;
|
activePrefix = null;
|
||||||
@@ -327,14 +355,20 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
}
|
}
|
||||||
private long currentLength(Connection connection, String bucket, String key) throws SQLException {
|
private long currentLength(Connection connection, String bucket, String key) throws SQLException {
|
||||||
try (PreparedStatement query = connection.prepareStatement("SELECT length FROM cluster_objects WHERE bucket=? AND object_key=?")) {
|
try (PreparedStatement query = connection.prepareStatement("SELECT length FROM cluster_objects WHERE bucket=? AND object_key=?")) {
|
||||||
query.setString(1, bucket); query.setString(2, key);
|
query.setString(1, bucket);
|
||||||
|
query.setString(2, key);
|
||||||
try (ResultSet result = query.executeQuery()) { return result.next() ? result.getLong(1) : -1; }
|
try (ResultSet result = query.executeQuery()) { return result.next() ? result.getLong(1) : -1; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
private static void bindObject(PreparedStatement update, Metadata data, UUID generation) throws SQLException {
|
private static void bindObject(PreparedStatement update, Metadata data, UUID generation) throws SQLException {
|
||||||
update.setString(1, data.bucket()); update.setString(2, data.key()); update.setObject(3, generation);
|
update.setString(1, data.bucket());
|
||||||
update.setLong(4, data.length()); update.setLong(5, data.modified()); update.setString(6, data.etag());
|
update.setString(2, data.key());
|
||||||
update.setBytes(7, data.sha256()); update.setString(8, data.contentType());
|
update.setObject(3, generation);
|
||||||
|
update.setLong(4, data.length());
|
||||||
|
update.setLong(5, data.modified());
|
||||||
|
update.setString(6, data.etag());
|
||||||
|
update.setBytes(7, data.sha256());
|
||||||
|
update.setString(8, data.contentType());
|
||||||
}
|
}
|
||||||
private static List<UUID> replicaIds(ResultSet result, int column) throws SQLException, IOException {
|
private static List<UUID> replicaIds(ResultSet result, int column) throws SQLException, IOException {
|
||||||
java.sql.Array value = result.getArray(column);
|
java.sql.Array value = result.getArray(column);
|
||||||
@@ -406,7 +440,10 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
healthy.add(id);
|
healthy.add(id);
|
||||||
healthyHosts.add(nodes.faultDomain(node, testNodeDomains));
|
healthyHosts.add(nodes.faultDomain(node, testNodeDomains));
|
||||||
}
|
}
|
||||||
if (copy == null) { unrecoverable++; continue; }
|
if (copy == null) {
|
||||||
|
unrecoverable++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
for (int node : PlacementPolicy.candidates(segment.id(), nodes, testNodeDomains)) {
|
for (int node : PlacementPolicy.candidates(segment.id(), nodes, testNodeDomains)) {
|
||||||
UUID host = nodes.faultDomain(node, testNodeDomains);
|
UUID host = nodes.faultDomain(node, testNodeDomains);
|
||||||
if (healthyHosts.contains(host)) continue;
|
if (healthyHosts.contains(host)) continue;
|
||||||
@@ -484,6 +521,9 @@ final class ClusterStore implements ObjectStorage {
|
|||||||
}
|
}
|
||||||
return current.read(buffer, offset, length);
|
return current.read(buffer, offset, length);
|
||||||
}
|
}
|
||||||
@Override public void close() { closed = true; current = null; }
|
@Override public void close() {
|
||||||
|
closed = true;
|
||||||
|
current = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -34,8 +34,10 @@ final class DiskStore implements ObjectStorage {
|
|||||||
|
|
||||||
DiskStore(Path root, long maxObject, long maxTotal) throws IOException {
|
DiskStore(Path root, long maxObject, long maxTotal) throws IOException {
|
||||||
this.root = root;
|
this.root = root;
|
||||||
objects = root.resolve("objects"); temporary = root.resolve("pending");
|
objects = root.resolve("objects");
|
||||||
this.maxObject = maxObject; this.maxTotal = maxTotal;
|
temporary = root.resolve("pending");
|
||||||
|
this.maxObject = maxObject;
|
||||||
|
this.maxTotal = maxTotal;
|
||||||
Arrays.setAll(locks, i -> new Object());
|
Arrays.setAll(locks, i -> new Object());
|
||||||
Files.createDirectories(root);
|
Files.createDirectories(root);
|
||||||
FileChannel channel = FileChannel.open(root.resolve(".process.lock"), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
|
FileChannel channel = FileChannel.open(root.resolve(".process.lock"), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
|
||||||
@@ -45,7 +47,8 @@ final class DiskStore implements ObjectStorage {
|
|||||||
try { acquired = channel.tryLock(); }
|
try { acquired = channel.tryLock(); }
|
||||||
catch (OverlappingFileLockException e) { throw new IOException("Data directory is already in use", e); }
|
catch (OverlappingFileLockException e) { throw new IOException("Data directory is already in use", e); }
|
||||||
if (acquired == null) throw new IOException("Data directory is already in use");
|
if (acquired == null) throw new IOException("Data directory is already in use");
|
||||||
Files.createDirectories(objects); Files.createDirectories(temporary);
|
Files.createDirectories(objects);
|
||||||
|
Files.createDirectories(temporary);
|
||||||
syncDirectory(root);
|
syncDirectory(root);
|
||||||
try (var paths = Files.list(temporary)) {
|
try (var paths = Files.list(temporary)) {
|
||||||
for (Path p : paths.toList()) if (p.getFileName().toString().endsWith(".part")) Files.delete(p);
|
for (Path p : paths.toList()) if (p.getFileName().toString().endsWith(".part")) Files.delete(p);
|
||||||
@@ -140,12 +143,15 @@ final class DiskStore implements ObjectStorage {
|
|||||||
long count = 0;
|
long count = 0;
|
||||||
try (OutputStream out = Files.newOutputStream(pending)) {
|
try (OutputStream out = Files.newOutputStream(pending)) {
|
||||||
out.write(new byte[headerLength]);
|
out.write(new byte[headerLength]);
|
||||||
byte[] buffer = new byte[65536]; int n;
|
byte[] buffer = new byte[65536];
|
||||||
|
int n;
|
||||||
while ((n = input.read(buffer)) != -1) {
|
while ((n = input.read(buffer)) != -1) {
|
||||||
count += n;
|
count += n;
|
||||||
if (count > length || count > maxObject)
|
if (count > length || count > maxObject)
|
||||||
throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size");
|
throw new StoreException(413, "EntityTooLarge", "Payload exceeds declared size");
|
||||||
sha.update(buffer, 0, n); md5.update(buffer, 0, n); out.write(buffer, 0, n);
|
sha.update(buffer, 0, n);
|
||||||
|
md5.update(buffer, 0, n);
|
||||||
|
out.write(buffer, 0, n);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (count != length) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length");
|
if (count != length) throw new StoreException(400, "IncompleteBody", "Payload length does not match Content-Length");
|
||||||
@@ -200,7 +206,10 @@ final class DiskStore implements ObjectStorage {
|
|||||||
try { input = new DataInputStream(Files.newInputStream(destination)); }
|
try { input = new DataInputStream(Files.newInputStream(destination)); }
|
||||||
catch (NoSuchFileException e) { throw new StoreException(404, "NoSuchKey", "Object not found"); }
|
catch (NoSuchFileException e) { throw new StoreException(404, "NoSuchKey", "Object not found"); }
|
||||||
try { return new OpenObject(readRecord(input).metadata(), input); }
|
try { return new OpenObject(readRecord(input).metadata(), input); }
|
||||||
catch (IOException e) { input.close(); throw e; }
|
catch (IOException e) {
|
||||||
|
input.close();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,10 +251,21 @@ final class DiskStore implements ObjectStorage {
|
|||||||
int at = key.indexOf(delimiter, prefix.length());
|
int at = key.indexOf(delimiter, prefix.length());
|
||||||
if (at >= 0) group = key.substring(0, at + delimiter.length());
|
if (at >= 0) group = key.substring(0, at + delimiter.length());
|
||||||
}
|
}
|
||||||
if (group != null && group.equals(activePrefix)) { lastKey = key; continue; }
|
if (group != null && group.equals(activePrefix)) {
|
||||||
if (entries.size() + prefixes.size() >= maxKeys) { truncated = true; break; }
|
lastKey = key;
|
||||||
if (group != null) { prefixes.add(group); activePrefix = group; }
|
continue;
|
||||||
else { entries.add(new ListedObject(key, meta)); activePrefix = null; }
|
}
|
||||||
|
if (entries.size() + prefixes.size() >= maxKeys) {
|
||||||
|
truncated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (group != null) {
|
||||||
|
prefixes.add(group);
|
||||||
|
activePrefix = group;
|
||||||
|
} else {
|
||||||
|
entries.add(new ListedObject(key, meta));
|
||||||
|
activePrefix = null;
|
||||||
|
}
|
||||||
lastKey = key;
|
lastKey = key;
|
||||||
}
|
}
|
||||||
return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated);
|
return new ListPage(entries, prefixes, truncated ? lastKey : null, truncated);
|
||||||
@@ -256,7 +276,8 @@ final class DiskStore implements ObjectStorage {
|
|||||||
if (magic != MAGIC_V1 && magic != MAGIC_V2) throw new IOException("Invalid object record");
|
if (magic != MAGIC_V1 && magic != MAGIC_V2) throw new IOException("Invalid object record");
|
||||||
long length = in.readLong(), modified = in.readLong();
|
long length = in.readLong(), modified = in.readLong();
|
||||||
byte[] md5 = new byte[16], sha = new byte[32];
|
byte[] md5 = new byte[16], sha = new byte[32];
|
||||||
in.readFully(md5); in.readFully(sha);
|
in.readFully(md5);
|
||||||
|
in.readFully(sha);
|
||||||
if (length < 0) throw new IOException("Invalid object record length");
|
if (length < 0) throw new IOException("Invalid object record length");
|
||||||
if (magic == MAGIC_V1)
|
if (magic == MAGIC_V1)
|
||||||
return new Record(new Metadata(length, modified, SigV4.hex(md5), sha,
|
return new Record(new Metadata(length, modified, SigV4.hex(md5), sha,
|
||||||
|
|||||||
@@ -34,8 +34,10 @@ public final class Main {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Main(ObjectStorage store, MultipartStorage multipart, SigV4 authentication, String bucket) {
|
Main(ObjectStorage store, MultipartStorage multipart, SigV4 authentication, String bucket) {
|
||||||
this.store = store; this.multipart = multipart;
|
this.store = store;
|
||||||
this.authentication = authentication; this.bucket = bucket;
|
this.multipart = multipart;
|
||||||
|
this.authentication = authentication;
|
||||||
|
this.bucket = bucket;
|
||||||
}
|
}
|
||||||
|
|
||||||
void handle(HttpExchange exchange) throws IOException {
|
void handle(HttpExchange exchange) throws IOException {
|
||||||
@@ -58,7 +60,10 @@ public final class Main {
|
|||||||
catch (Exception error) {
|
catch (Exception error) {
|
||||||
System.err.println("ObjectStore request failed: " + requestId + " " + error.getClass().getSimpleName());
|
System.err.println("ObjectStore request failed: " + requestId + " " + error.getClass().getSimpleName());
|
||||||
sendError(exchange, 500, "InternalError", "Storage operation failed", requestId);
|
sendError(exchange, 500, "InternalError", "Storage operation failed", requestId);
|
||||||
} finally { if (admitted) slots.release(); exchange.close(); }
|
} finally {
|
||||||
|
if (admitted) slots.release();
|
||||||
|
exchange.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean handleStatus(HttpExchange exchange) throws IOException {
|
private boolean handleStatus(HttpExchange exchange) throws IOException {
|
||||||
@@ -321,7 +326,8 @@ public final class Main {
|
|||||||
else {
|
else {
|
||||||
object.stream().skipNBytes(range.start());
|
object.stream().skipNBytes(range.start());
|
||||||
exchange.sendResponseHeaders(status, range.length());
|
exchange.sendResponseHeaders(status, range.length());
|
||||||
byte[] buffer = new byte[65536]; long left = range.length();
|
byte[] buffer = new byte[65536];
|
||||||
|
long left = range.length();
|
||||||
while (left > 0) {
|
while (left > 0) {
|
||||||
int n = object.stream().read(buffer, 0, (int) Math.min(buffer.length, left));
|
int n = object.stream().read(buffer, 0, (int) Math.min(buffer.length, left));
|
||||||
if (n < 0) throw new IOException("Object body ended before its recorded length");
|
if (n < 0) throw new IOException("Object body ended before its recorded length");
|
||||||
@@ -345,7 +351,8 @@ public final class Main {
|
|||||||
if (parts[0].isEmpty()) {
|
if (parts[0].isEmpty()) {
|
||||||
long suffix = Long.parseLong(parts[1]);
|
long suffix = Long.parseLong(parts[1]);
|
||||||
if (suffix == 0) throw new NumberFormatException();
|
if (suffix == 0) throw new NumberFormatException();
|
||||||
start = Math.max(0, size - suffix); end = size - 1;
|
start = Math.max(0, size - suffix);
|
||||||
|
end = size - 1;
|
||||||
} else {
|
} else {
|
||||||
start = Long.parseLong(parts[0]);
|
start = Long.parseLong(parts[0]);
|
||||||
end = parts[1].isEmpty() ? size - 1 : Math.min(Long.parseLong(parts[1]), size - 1);
|
end = parts[1].isEmpty() ? size - 1 : Math.min(Long.parseLong(parts[1]), size - 1);
|
||||||
@@ -494,7 +501,8 @@ public final class Main {
|
|||||||
? new InetSocketAddress(env.getOrDefault("BIND_ADDRESS", "127.0.0.1"), port)
|
? new InetSocketAddress(env.getOrDefault("BIND_ADDRESS", "127.0.0.1"), port)
|
||||||
: new InetSocketAddress(port), 64);
|
: new InetSocketAddress(port), 64);
|
||||||
var executor = Executors.newVirtualThreadPerTaskExecutor();
|
var executor = Executors.newVirtualThreadPerTaskExecutor();
|
||||||
server.setExecutor(executor); server.createContext("/", app::handle);
|
server.setExecutor(executor);
|
||||||
|
server.createContext("/", app::handle);
|
||||||
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
|
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
|
||||||
server.stop(5);
|
server.stop(5);
|
||||||
executor.close();
|
executor.close();
|
||||||
@@ -526,7 +534,10 @@ public final class Main {
|
|||||||
String[] units = {"KiB", "MiB", "GiB", "TiB"};
|
String[] units = {"KiB", "MiB", "GiB", "TiB"};
|
||||||
double value = bytes;
|
double value = bytes;
|
||||||
int unit = -1;
|
int unit = -1;
|
||||||
do { value /= 1024; unit++; } while (value >= 1024 && unit < units.length - 1);
|
do {
|
||||||
|
value /= 1024;
|
||||||
|
unit++;
|
||||||
|
} while (value >= 1024 && unit < units.length - 1);
|
||||||
return String.format(Locale.ROOT, "%.1f %s", value, units[unit]);
|
return String.format(Locale.ROOT, "%.1f %s", value, units[unit]);
|
||||||
}
|
}
|
||||||
private static String required(Map<String, String> env, String key) {
|
private static String required(Map<String, String> env, String key) {
|
||||||
|
|||||||
@@ -97,11 +97,14 @@ final class MultipartStore implements MultipartStorage {
|
|||||||
MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
|
MessageDigest sha = digest("SHA-256"), md5 = digest("MD5");
|
||||||
long count = 0;
|
long count = 0;
|
||||||
try (var output = Files.newOutputStream(pending)) {
|
try (var output = Files.newOutputStream(pending)) {
|
||||||
byte[] buffer = new byte[65536]; int n;
|
byte[] buffer = new byte[65536];
|
||||||
|
int n;
|
||||||
while ((n = input.read(buffer)) != -1) {
|
while ((n = input.read(buffer)) != -1) {
|
||||||
count += n;
|
count += n;
|
||||||
if (count > length) throw new StoreException(413, "EntityTooLarge", "Part exceeds declared size");
|
if (count > length) throw new StoreException(413, "EntityTooLarge", "Part exceeds declared size");
|
||||||
sha.update(buffer, 0, n); md5.update(buffer, 0, n); output.write(buffer, 0, n);
|
sha.update(buffer, 0, n);
|
||||||
|
md5.update(buffer, 0, n);
|
||||||
|
output.write(buffer, 0, n);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (count != length) throw new StoreException(400, "IncompleteBody", "Part length does not match Content-Length");
|
if (count != length) throw new StoreException(400, "IncompleteBody", "Part length does not match Content-Length");
|
||||||
@@ -124,7 +127,8 @@ final class MultipartStore implements MultipartStorage {
|
|||||||
throw new StoreException(400, "InvalidPart", "No valid parts supplied");
|
throw new StoreException(400, "InvalidPart", "No valid parts supplied");
|
||||||
MessageDigest sha = digest("SHA-256");
|
MessageDigest sha = digest("SHA-256");
|
||||||
List<Path> paths = new ArrayList<>();
|
List<Path> paths = new ArrayList<>();
|
||||||
long total = 0; int last = 0;
|
long total = 0;
|
||||||
|
int last = 0;
|
||||||
for (Part part : parts) {
|
for (Part part : parts) {
|
||||||
if (part.number() <= last || part.number() > 10000)
|
if (part.number() <= last || part.number() > 10000)
|
||||||
throw new StoreException(400, "InvalidPartOrder", "Parts must be in ascending order");
|
throw new StoreException(400, "InvalidPartOrder", "Parts must be in ascending order");
|
||||||
@@ -136,8 +140,12 @@ final class MultipartStore implements MultipartStorage {
|
|||||||
if (total > store.maxObject()) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit");
|
if (total > store.maxObject()) throw new StoreException(413, "EntityTooLarge", "Object exceeds the configured size limit");
|
||||||
MessageDigest md5 = digest("MD5");
|
MessageDigest md5 = digest("MD5");
|
||||||
try (var input = Files.newInputStream(file)) {
|
try (var input = Files.newInputStream(file)) {
|
||||||
byte[] buffer = new byte[65536]; int n;
|
byte[] buffer = new byte[65536];
|
||||||
while ((n = input.read(buffer)) != -1) { sha.update(buffer, 0, n); md5.update(buffer, 0, n); }
|
int n;
|
||||||
|
while ((n = input.read(buffer)) != -1) {
|
||||||
|
sha.update(buffer, 0, n);
|
||||||
|
md5.update(buffer, 0, n);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (!SigV4.hex(md5.digest()).equals(part.etag().replace("\"", "")))
|
if (!SigV4.hex(md5.digest()).equals(part.etag().replace("\"", "")))
|
||||||
throw new StoreException(400, "InvalidPart", "Part ETag mismatch");
|
throw new StoreException(400, "InvalidPart", "Part ETag mismatch");
|
||||||
@@ -209,7 +217,8 @@ final class MultipartStore implements MultipartStorage {
|
|||||||
}
|
}
|
||||||
int n = current.read(buffer, offset, length);
|
int n = current.read(buffer, offset, length);
|
||||||
if (n >= 0) return n;
|
if (n >= 0) return n;
|
||||||
current.close(); current = null;
|
current.close();
|
||||||
|
current = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@Override public void close() throws IOException { if (current != null) current.close(); }
|
@Override public void close() throws IOException { if (current != null) current.close(); }
|
||||||
|
|||||||
@@ -43,7 +43,9 @@ final class NodeClient {
|
|||||||
List<Node> nodes() { return nodes; }
|
List<Node> nodes() { return nodes; }
|
||||||
Node node(int index) { return nodes.get(index); }
|
Node node(int index) { return nodes.get(index); }
|
||||||
int index(UUID id) {
|
int index(UUID id) {
|
||||||
for (int i = 0; i < nodes.size(); i++) if (nodes.get(i).id().equals(id)) return i;
|
for (int i = 0; i < nodes.size(); i++) {
|
||||||
|
if (nodes.get(i).id().equals(id)) return i;
|
||||||
|
}
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
UUID faultDomain(int index, boolean testNodeDomains) {
|
UUID faultDomain(int index, boolean testNodeDomains) {
|
||||||
|
|||||||
@@ -52,7 +52,8 @@ final class NodeRegistry {
|
|||||||
connection.commit();
|
connection.commit();
|
||||||
return new NodeClient.Node(identity.nodeId(), identity.hostId(), url);
|
return new NodeClient.Node(identity.nodeId(), identity.hostId(), url);
|
||||||
} catch (SQLException | IOException error) {
|
} catch (SQLException | IOException error) {
|
||||||
try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
|
try { connection.rollback(); }
|
||||||
|
catch (SQLException rollback) { error.addSuppressed(rollback); }
|
||||||
if (error instanceof IOException io) throw io;
|
if (error instanceof IOException io) throw io;
|
||||||
throw new IOException("Node registration failed", error);
|
throw new IOException("Node registration failed", error);
|
||||||
} finally {
|
} finally {
|
||||||
@@ -77,7 +78,8 @@ final class NodeRegistry {
|
|||||||
connection.commit();
|
connection.commit();
|
||||||
return nodes;
|
return nodes;
|
||||||
} catch (SQLException | IOException | RuntimeException error) {
|
} catch (SQLException | IOException | RuntimeException error) {
|
||||||
try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
|
try { connection.rollback(); }
|
||||||
|
catch (SQLException rollback) { error.addSuppressed(rollback); }
|
||||||
if (error instanceof IOException io) throw io;
|
if (error instanceof IOException io) throw io;
|
||||||
if (error instanceof SQLException sql) throw new IOException("Node registry check failed", sql);
|
if (error instanceof SQLException sql) throw new IOException("Node registry check failed", sql);
|
||||||
throw (RuntimeException) error;
|
throw (RuntimeException) error;
|
||||||
|
|||||||
@@ -60,7 +60,8 @@ final class SchemaMigrator {
|
|||||||
connection.commit();
|
connection.commit();
|
||||||
return format;
|
return format;
|
||||||
} catch (SQLException | IOException error) {
|
} catch (SQLException | IOException error) {
|
||||||
try { connection.rollback(); } catch (SQLException rollback) { error.addSuppressed(rollback); }
|
try { connection.rollback(); }
|
||||||
|
catch (SQLException rollback) { error.addSuppressed(rollback); }
|
||||||
if (error instanceof IOException io) throw io;
|
if (error instanceof IOException io) throw io;
|
||||||
throw new IOException("Metadata schema migration failed", error);
|
throw new IOException("Metadata schema migration failed", error);
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -24,7 +24,10 @@ final class SigV4 {
|
|||||||
private final Clock clock;
|
private final Clock clock;
|
||||||
|
|
||||||
SigV4(String accessKey, String secretKey, String region, Clock clock) {
|
SigV4(String accessKey, String secretKey, String region, Clock clock) {
|
||||||
this.accessKey = accessKey; this.secretKey = secretKey; this.region = region; this.clock = clock;
|
this.accessKey = accessKey;
|
||||||
|
this.secretKey = secretKey;
|
||||||
|
this.region = region;
|
||||||
|
this.clock = clock;
|
||||||
}
|
}
|
||||||
|
|
||||||
String verify(String method, URI uri, Headers headers) {
|
String verify(String method, URI uri, Headers headers) {
|
||||||
@@ -111,17 +114,25 @@ final class SigV4 {
|
|||||||
|
|
||||||
static String decode(String value) {
|
static String decode(String value) {
|
||||||
try {
|
try {
|
||||||
var bytes=new java.io.ByteArrayOutputStream();
|
var bytes = new java.io.ByteArrayOutputStream();
|
||||||
for(int i=0;i<value.length();){
|
for (int i = 0; i < value.length();) {
|
||||||
if(value.charAt(i)=='%'){
|
if (value.charAt(i) == '%') {
|
||||||
if(i+2>=value.length())throw new IllegalArgumentException();
|
if (i + 2 >= value.length()) throw new IllegalArgumentException();
|
||||||
int hi=Character.digit(value.charAt(i+1),16),lo=Character.digit(value.charAt(i+2),16);
|
int hi = Character.digit(value.charAt(i + 1), 16);
|
||||||
if(hi<0||lo<0)throw new IllegalArgumentException();
|
int lo = Character.digit(value.charAt(i + 2), 16);
|
||||||
bytes.write((hi<<4)|lo);i+=3;
|
if (hi < 0 || lo < 0) throw new IllegalArgumentException();
|
||||||
}else{int point=value.codePointAt(i);bytes.writeBytes(new String(Character.toChars(point)).getBytes(StandardCharsets.UTF_8));i+=Character.charCount(point);}
|
bytes.write((hi << 4) | lo);
|
||||||
|
i += 3;
|
||||||
|
} else {
|
||||||
|
int point = value.codePointAt(i);
|
||||||
|
bytes.writeBytes(new String(Character.toChars(point)).getBytes(StandardCharsets.UTF_8));
|
||||||
|
i += Character.charCount(point);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return StandardCharsets.UTF_8.newDecoder().onMalformedInput(java.nio.charset.CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes.toByteArray())).toString();
|
return StandardCharsets.UTF_8.newDecoder().onMalformedInput(java.nio.charset.CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes.toByteArray())).toString();
|
||||||
}catch(IllegalArgumentException|java.nio.charset.CharacterCodingException e){throw new StoreException(400,"InvalidURI","Malformed URI encoding");}
|
} catch (IllegalArgumentException | java.nio.charset.CharacterCodingException e) {
|
||||||
|
throw new StoreException(400, "InvalidURI", "Malformed URI encoding");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static String encode(String value, boolean keepSlash) {
|
static String encode(String value, boolean keepSlash) {
|
||||||
@@ -146,10 +157,17 @@ final class SigV4 {
|
|||||||
return hmac(hmac(hmac(hmac(("AWS4"+secret).getBytes(StandardCharsets.UTF_8),date),region),"s3"),"aws4_request");
|
return hmac(hmac(hmac(hmac(("AWS4"+secret).getBytes(StandardCharsets.UTF_8),date),region),"s3"),"aws4_request");
|
||||||
}
|
}
|
||||||
static byte[] hmac(byte[] key, String text) {
|
static byte[] hmac(byte[] key, String text) {
|
||||||
try { Mac mac=Mac.getInstance("HmacSHA256");mac.init(new SecretKeySpec(key,"HmacSHA256"));return mac.doFinal(text.getBytes(StandardCharsets.UTF_8)); }
|
try {
|
||||||
|
Mac mac = Mac.getInstance("HmacSHA256");
|
||||||
|
mac.init(new SecretKeySpec(key, "HmacSHA256"));
|
||||||
|
return mac.doFinal(text.getBytes(StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
catch (java.security.GeneralSecurityException e) { throw new IllegalStateException(e); }
|
catch (java.security.GeneralSecurityException e) { throw new IllegalStateException(e); }
|
||||||
}
|
}
|
||||||
static byte[] hash(byte[] data) { try {return MessageDigest.getInstance("SHA-256").digest(data);}catch(java.security.NoSuchAlgorithmException e){throw new IllegalStateException(e);} }
|
static byte[] hash(byte[] data) {
|
||||||
|
try { return MessageDigest.getInstance("SHA-256").digest(data); }
|
||||||
|
catch (java.security.NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
|
||||||
|
}
|
||||||
static String hex(byte[] data) { return HexFormat.of().formatHex(data); }
|
static String hex(byte[] data) { return HexFormat.of().formatHex(data); }
|
||||||
private static void denied(String message) { throw new StoreException(403,"AccessDenied",message); }
|
private static void denied(String message) { throw new StoreException(403,"AccessDenied",message); }
|
||||||
}
|
}
|
||||||
@@ -21,7 +21,9 @@ public final class ClusterIntegrationTest {
|
|||||||
switch (args[0]) {
|
switch (args[0]) {
|
||||||
case "basic" -> {
|
case "basic" -> {
|
||||||
byte[] large = new byte[ClusterNode.MAX_SEGMENT + 37];
|
byte[] large = new byte[ClusterNode.MAX_SEGMENT + 37];
|
||||||
for (int i = 0; i < large.length; i++) large[i] = (byte) (i * 31);
|
for (int i = 0; i < large.length; i++) {
|
||||||
|
large[i] = (byte) (i * 31);
|
||||||
|
}
|
||||||
String largeKey = "cluster-test/large";
|
String largeKey = "cluster-test/large";
|
||||||
put(store, bucket, largeKey, large, false);
|
put(store, bucket, largeKey, large, false);
|
||||||
try (var opened = store.open(bucket, largeKey)) {
|
try (var opened = store.open(bucket, largeKey)) {
|
||||||
@@ -39,7 +41,10 @@ public final class ClusterIntegrationTest {
|
|||||||
"Overwrite was not visible");
|
"Overwrite was not visible");
|
||||||
}
|
}
|
||||||
store.delete(bucket, largeKey);
|
store.delete(bucket, largeKey);
|
||||||
try { store.open(bucket, largeKey); throw new AssertionError("Deleted object remained visible"); }
|
try {
|
||||||
|
store.open(bucket, largeKey);
|
||||||
|
throw new AssertionError("Deleted object remained visible");
|
||||||
|
}
|
||||||
catch (StoreException error) { require(error.status == 404, "Wrong missing-object status"); }
|
catch (StoreException error) { require(error.status == 404, "Wrong missing-object status"); }
|
||||||
put(store, bucket, KEY, stable, false);
|
put(store, bucket, KEY, stable, false);
|
||||||
require(store.ready(), "Healthy cluster is not ready");
|
require(store.ready(), "Healthy cluster is not ready");
|
||||||
@@ -63,7 +68,10 @@ public final class ClusterIntegrationTest {
|
|||||||
put(store, bucket, "cluster-test/rejected", new byte[]{1}, false);
|
put(store, bucket, "cluster-test/rejected", new byte[]{1}, false);
|
||||||
throw new AssertionError("Write succeeded with only one node");
|
throw new AssertionError("Write succeeded with only one node");
|
||||||
} catch (StoreException error) { require(error.status == 503, "Wrong unavailable status"); }
|
} catch (StoreException error) { require(error.status == 503, "Wrong unavailable status"); }
|
||||||
try { store.open(bucket, "cluster-test/rejected"); throw new AssertionError("Failed write became visible"); }
|
try {
|
||||||
|
store.open(bucket, "cluster-test/rejected");
|
||||||
|
throw new AssertionError("Failed write became visible");
|
||||||
|
}
|
||||||
catch (StoreException error) { require(error.status == 404, "Partial object became visible"); }
|
catch (StoreException error) { require(error.status == 404, "Partial object became visible"); }
|
||||||
System.out.println("Cluster quorum-loss test passed");
|
System.out.println("Cluster quorum-loss test passed");
|
||||||
}
|
}
|
||||||
@@ -82,13 +90,18 @@ public final class ClusterIntegrationTest {
|
|||||||
var executor = java.util.concurrent.Executors.newFixedThreadPool(2);
|
var executor = java.util.concurrent.Executors.newFixedThreadPool(2);
|
||||||
try {
|
try {
|
||||||
var a = executor.submit(() -> {
|
var a = executor.submit(() -> {
|
||||||
start.await(); put(store, bucket, key, first, false); return null;
|
start.await();
|
||||||
|
put(store, bucket, key, first, false);
|
||||||
|
return null;
|
||||||
});
|
});
|
||||||
var b = executor.submit(() -> {
|
var b = executor.submit(() -> {
|
||||||
start.await(); put(store, bucket, key, second, false); return null;
|
start.await();
|
||||||
|
put(store, bucket, key, second, false);
|
||||||
|
return null;
|
||||||
});
|
});
|
||||||
start.countDown();
|
start.countDown();
|
||||||
a.get(); b.get();
|
a.get();
|
||||||
|
b.get();
|
||||||
try (var opened = store.open(bucket, key)) {
|
try (var opened = store.open(bucket, key)) {
|
||||||
byte[] actual = opened.stream().readAllBytes();
|
byte[] actual = opened.stream().readAllBytes();
|
||||||
require(Arrays.equals(actual, first) || Arrays.equals(actual, second),
|
require(Arrays.equals(actual, first) || Arrays.equals(actual, second),
|
||||||
|
|||||||
@@ -31,7 +31,9 @@ public final class ClusterMigrationTest {
|
|||||||
if (args[0].equals("create")) {
|
if (args[0].equals("create")) {
|
||||||
UUID segment = UUID.randomUUID();
|
UUID segment = UUID.randomUUID();
|
||||||
byte[] hash = SigV4.hash(DATA);
|
byte[] hash = SigV4.hash(DATA);
|
||||||
for (int i = 0; i < nodes.count(); i++) nodes.put(i, segment, DATA, hash);
|
for (int i = 0; i < nodes.count(); i++) {
|
||||||
|
nodes.put(i, segment, DATA, hash);
|
||||||
|
}
|
||||||
try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
|
try (var connection = DriverManager.getConnection(env.get("POSTGRES_JDBC_URL"),
|
||||||
env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
|
env.get("POSTGRES_USER"), env.get("POSTGRES_PASSWORD"));
|
||||||
var statement = connection.createStatement()) {
|
var statement = connection.createStatement()) {
|
||||||
@@ -40,18 +42,28 @@ public final class ClusterMigrationTest {
|
|||||||
statement.execute("CREATE TABLE cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))");
|
statement.execute("CREATE TABLE cluster_segments (generation uuid NOT NULL, ordinal integer NOT NULL, segment_id uuid NOT NULL, length integer NOT NULL, sha256 bytea NOT NULL, replicas text NOT NULL, PRIMARY KEY (generation, ordinal))");
|
||||||
statement.execute("CREATE TABLE cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))");
|
statement.execute("CREATE TABLE cluster_tombstones (bucket text NOT NULL, object_key text COLLATE \"C\" NOT NULL, generation uuid NOT NULL, deleted_at bigint NOT NULL, PRIMARY KEY (bucket, object_key))");
|
||||||
try (var insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, ?)")) {
|
try (var insert = connection.prepareStatement("INSERT INTO cluster_usage VALUES (?, ?)")) {
|
||||||
insert.setString(1, bucket); insert.setLong(2, DATA.length); insert.executeUpdate();
|
insert.setString(1, bucket);
|
||||||
|
insert.setLong(2, DATA.length);
|
||||||
|
insert.executeUpdate();
|
||||||
}
|
}
|
||||||
UUID generation = UUID.randomUUID();
|
UUID generation = UUID.randomUUID();
|
||||||
try (var insert = connection.prepareStatement("INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) {
|
try (var insert = connection.prepareStatement("INSERT INTO cluster_objects VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) {
|
||||||
insert.setString(1, bucket); insert.setString(2, KEY); insert.setObject(3, generation);
|
insert.setString(1, bucket);
|
||||||
insert.setLong(4, DATA.length); insert.setLong(5, System.currentTimeMillis());
|
insert.setString(2, KEY);
|
||||||
|
insert.setObject(3, generation);
|
||||||
|
insert.setLong(4, DATA.length);
|
||||||
|
insert.setLong(5, System.currentTimeMillis());
|
||||||
insert.setString(6, HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(DATA)));
|
insert.setString(6, HexFormat.of().formatHex(MessageDigest.getInstance("MD5").digest(DATA)));
|
||||||
insert.setBytes(7, hash); insert.setString(8, "text/plain"); insert.executeUpdate();
|
insert.setBytes(7, hash);
|
||||||
|
insert.setString(8, "text/plain");
|
||||||
|
insert.executeUpdate();
|
||||||
}
|
}
|
||||||
try (var insert = connection.prepareStatement("INSERT INTO cluster_segments VALUES (?, 0, ?, ?, ?, '0,1,2')")) {
|
try (var insert = connection.prepareStatement("INSERT INTO cluster_segments VALUES (?, 0, ?, ?, ?, '0,1,2')")) {
|
||||||
insert.setObject(1, generation); insert.setObject(2, segment);
|
insert.setObject(1, generation);
|
||||||
insert.setInt(3, DATA.length); insert.setBytes(4, hash); insert.executeUpdate();
|
insert.setObject(2, segment);
|
||||||
|
insert.setInt(3, DATA.length);
|
||||||
|
insert.setBytes(4, hash);
|
||||||
|
insert.executeUpdate();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
System.out.println("Legacy cluster fixture created");
|
System.out.println("Legacy cluster fixture created");
|
||||||
|
|||||||
@@ -142,7 +142,8 @@ public final class HttpTest {
|
|||||||
"?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
"?partNumber=1&uploadId=" + upload), "PUT", first, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
||||||
var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie +
|
var partTwo = client.send(signedUri(URI.create(base + "/objects/" + movie +
|
||||||
"?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
"?partNumber=2&uploadId=" + upload), "PUT", second, Map.of()), HttpResponse.BodyHandlers.ofByteArray());
|
||||||
status(200, partOne); status(200, partTwo);
|
status(200, partOne);
|
||||||
|
status(200, partTwo);
|
||||||
String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
|
String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" +
|
||||||
partOne.headers().firstValue("etag").orElseThrow() +
|
partOne.headers().firstValue("etag").orElseThrow() +
|
||||||
"</ETag></Part><Part><PartNumber>2</PartNumber><ETag>" +
|
"</ETag></Part><Part><PartNumber>2</PartNumber><ETag>" +
|
||||||
|
|||||||
@@ -9,25 +9,33 @@ import java.util.List;
|
|||||||
public final class StoreTest {
|
public final class StoreTest {
|
||||||
interface Operation {void run() throws Exception;}
|
interface Operation {void run() throws Exception;}
|
||||||
static void fails(int status,Operation operation)throws Exception{
|
static void fails(int status,Operation operation)throws Exception{
|
||||||
try{operation.run();throw new AssertionError("Expected "+status);}catch(StoreException error){if(error.status!=status)throw error;}
|
try {
|
||||||
|
operation.run();
|
||||||
|
throw new AssertionError("Expected " + status);
|
||||||
|
} catch (StoreException error) {
|
||||||
|
if (error.status != status) throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
|
static ObjectStorage.Metadata put(DiskStore store,String key,byte[] body,boolean only)throws Exception{
|
||||||
return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream");
|
return store.put("test",key,new ByteArrayInputStream(body),body.length,SigV4.hex(SigV4.hash(body)),null,only,"application/octet-stream");
|
||||||
}
|
}
|
||||||
private static void testSignature() throws Exception {
|
private static void testSignature() throws Exception {
|
||||||
var headers=new com.sun.net.httpserver.Headers();
|
var headers=new com.sun.net.httpserver.Headers();
|
||||||
headers.set("host","examplebucket.s3.amazonaws.com");headers.set("range","bytes=0-9");
|
headers.set("host","examplebucket.s3.amazonaws.com");
|
||||||
|
headers.set("range","bytes=0-9");
|
||||||
headers.set("x-amz-date","20130524T000000Z");
|
headers.set("x-amz-date","20130524T000000Z");
|
||||||
headers.set("x-amz-content-sha256","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
headers.set("x-amz-content-sha256","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||||
headers.set("authorization","AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41");
|
headers.set("authorization","AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41");
|
||||||
var clock=java.time.Clock.fixed(java.time.Instant.parse("2013-05-24T00:00:00Z"),java.time.ZoneOffset.UTC);
|
var clock=java.time.Clock.fixed(java.time.Instant.parse("2013-05-24T00:00:00Z"),java.time.ZoneOffset.UTC);
|
||||||
var auth=new SigV4("AKIAIOSFODNN7EXAMPLE","wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","us-east-1",clock);
|
var auth=new SigV4("AKIAIOSFODNN7EXAMPLE","wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","us-east-1",clock);
|
||||||
var uri=java.net.URI.create("/test.txt");auth.verify("GET",uri,headers);
|
var uri=java.net.URI.create("/test.txt");
|
||||||
|
auth.verify("GET",uri,headers);
|
||||||
fails(403,()->auth.verify("GET",java.net.URI.create("/other.txt"),headers));
|
fails(403,()->auth.verify("GET",java.net.URI.create("/other.txt"),headers));
|
||||||
fails(403,()->auth.verify("DELETE",uri,headers));
|
fails(403,()->auth.verify("DELETE",uri,headers));
|
||||||
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",clock).verify("GET",uri,headers));
|
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",clock).verify("GET",uri,headers));
|
||||||
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers));
|
fails(403,()->new SigV4("AKIAIOSFODNN7EXAMPLE","wrong","us-east-1",java.time.Clock.systemUTC()).verify("GET",uri,headers));
|
||||||
headers.add("host","duplicate");fails(403,()->auth.verify("GET",uri,headers));
|
headers.add("host","duplicate");
|
||||||
|
fails(403,()->auth.verify("GET",uri,headers));
|
||||||
System.out.println("SigV4 official vector and tampering tests passed");
|
System.out.println("SigV4 official vector and tampering tests passed");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,7 +67,8 @@ public final class StoreTest {
|
|||||||
try(var restarted=new DiskStore(root,8,10)){
|
try(var restarted=new DiskStore(root,8,10)){
|
||||||
try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
|
try(var obj=restarted.open("test","../nested/☾")){if(obj.stream().read()!=9)throw new AssertionError("Persistence");}
|
||||||
if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
|
if(restarted.list("test","","",100,null).objects().size()!=2)throw new AssertionError("Index persistence");
|
||||||
restarted.delete("test","../nested/☾");restarted.delete("test","../nested/☾");
|
restarted.delete("test","../nested/☾");
|
||||||
|
restarted.delete("test","../nested/☾");
|
||||||
fails(404,()->restarted.open("test","../nested/☾"));
|
fails(404,()->restarted.open("test","../nested/☾"));
|
||||||
var uploads=new MultipartStore(restarted);
|
var uploads=new MultipartStore(restarted);
|
||||||
String upload=uploads.create("test","from-parts","text/plain");
|
String upload=uploads.create("test","from-parts","text/plain");
|
||||||
|
|||||||
Reference in new issue
Block a user