Skip to content

Fix S3 suffix byte-range requests when reading sharded chunks #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
konstibob wants to merge 2 commits into
base: main
Choose a base branch
from
+42 −9
Open

Fix S3 suffix byte-range requests when reading sharded chunks #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
726dea6
fixed checksum Error
konstibob Jul 2, 2026
d406e5e
switched testcase location
konstibob Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions src/main/java/dev/zarr/zarrjava/store/S3Store.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,29 @@
package dev.zarr.zarrjava.store;

import dev.zarr.zarrjava.utils.Utils;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.stream.Stream;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import dev.zarr.zarrjava.utils.Utils;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CommonPrefix;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;

public class S3Store implements Store, Store.ListableStore {

@Nonnull
Expand Down Expand Up @@ -82,7 +93,9 @@ public ByteBuffer get(String[] keys, long start) {
GetObjectRequest req = GetObjectRequest.builder()
.bucket(bucketName)
.key(resolveKeys(keys))
.range(String.format("bytes=%d-", start))
.range(start < 0 //dependant on where we start either fetch last or first bytes
? String.format("bytes=%d", start)
: String.format("bytes=%d-", start))
.build();
return get(req);
}
Expand Down
20 changes: 20 additions & 0 deletions src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,26 @@ public void testGet() {
Assertions.assertEquals(10, bufferWithStartAndEnd.remaining());
}

@Test
public void testReadSuffix() {
StoreHandle storeHandle = storeHandleWithData();
ByteBuffer fullBuffer = storeHandle.read();
long size = fullBuffer.remaining();
if (size < 20) {
Assertions.fail("Store size is too small to test suffix read");
}

ByteBuffer suffixBuffer = storeHandle.read(-10);
Assertions.assertEquals(10, suffixBuffer.remaining());

byte[] expectedBytes = new byte[10];
fullBuffer.position((int) (size - 10));
fullBuffer.get(expectedBytes);
byte[] actualBytes = new byte[10];
suffixBuffer.get(actualBytes);
Assertions.assertArrayEquals(expectedBytes, actualBytes);
}

@Override
StoreHandle storeHandleWithData() {
return storeHandle.resolve("zarr.json");
Expand Down
Loading