From 94f2b974619e6e0db6e66f345415310d372c2813 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 21 Jul 2026 14:52:59 -0400 Subject: [PATCH 1/7] Library/version updates --- README.md | 14 ++- pom.xml | 85 ++++++++++++------- .../sead/uploader/dataverse/DVUploader.java | 2 +- 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 6242114..7ce0e4d 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,14 @@ For Clowder/SEAD-specific information, see https://opensource.ncsa.illinois.edu/ Build: - mvn clean compile assembly:single - - Usage: See wiki: https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse +Run the following command to build the project and create the executable JAR: + +```bash +mvn clean package -DskipTests +``` + +This will produce: +- `target/DVUploader-1.4.0.jar`: Standard library JAR. +- `target/DVUploader-v1.4.0.jar`: Executable "fat" JAR containing all dependencies. + +Usage: See wiki: https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse diff --git a/pom.xml b/pom.xml index d3d2a69..5929de1 100644 --- a/pom.xml +++ b/pom.xml @@ -4,25 +4,59 @@ 4.0.0 DVUploader DVUploader - 1.3.0-beta + 1.4.0 UTF-8 + 21 + 21 + 4.5.14 + 4.4.16 + 2.22.1 + 2.22 + 1.22.0 + 2.22.0 + 20260522 + 1.4.0 + 2.0.1 + 1.3.1 src/main/java maven-compiler-plugin - 3.11.0 + 3.13.0 - 1.8 - 1.8 + ${maven.compiler.source} + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.1 + + + + true + org.sead.uploader.dataverse.DVUploader + + org.apache.maven.plugins maven-assembly-plugin - 3.6.0 + 3.7.1 + + + make-assembly + package + + single + + + @@ -42,67 +76,52 @@ org.apache.httpcomponents httpclient - 4.5.13 - - - org.apache.httpcomponents - httpclient-cache - 4.5.13 + ${httpclient.version} org.apache.httpcomponents httpmime - 4.5.13 + ${httpclient.version} org.apache.httpcomponents - fluent-hc - 4.5.13 + httpcore + ${httpcore.version} - org.apache.httpcomponents - httpcore - 4.4.15 + commons-logging + commons-logging + ${commons-logging.version} commons-codec commons-codec - 1.15 - - - com.fasterxml.jackson.core - jackson-core - 2.18.2 + ${commons-codec.version} com.fasterxml.jackson.core jackson-databind - 2.18.2 + ${jackson.databind.version} org.json json - 20250107 + ${json.version} commons-io commons-io - 2.18.0 - - - org.apache.commons - commons-collections4 - 4.4 + ${commons-io.version} com.apicatalog titanium-json-ld - 1.4.0 + ${titanium-json-ld.version} org.glassfish jakarta.json - 2.0.1 + ${jakarta.json.version} diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index 334e704..3b7a471 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -154,7 +154,7 @@ public static void main(String args[]) throws Exception { private static void usage() { println("\nUsage:"); - println(" java -jar DVUploader-1.2.0.jar -server= -key= -did= "); + println(" java -jar DVUploader-v1.4.0.jar -server= -key= -did= "); println("\n where:"); println(" = the URL of the server to upload to, e.g. https://datverse.tdl.org"); From a663cafce456a57da6c2c13d16e5992a6e91feeb Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 23 Jul 2026 13:51:22 -0400 Subject: [PATCH 2/7] initial tests --- .gitignore | 4 +- README.md | 16 ++ pom.xml | 17 ++ .../org/sead/uploader/AbstractUploader.java | 9 + .../sead/uploader/dataverse/DVUploader.java | 12 +- .../uploader/dataverse/DVUploaderTest.java | 198 ++++++++++++++++++ test.properties.example | 5 + 7 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java create mode 100644 test.properties.example diff --git a/.gitignore b/.gitignore index a6f89c2..bead6a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -/target/ \ No newline at end of file +/target/ +test.properties +DVUploaderLog_*.log \ No newline at end of file diff --git a/README.md b/README.md index 7ce0e4d..e9d630f 100644 --- a/README.md +++ b/README.md @@ -29,4 +29,20 @@ This will produce: - `target/DVUploader-1.4.0.jar`: Standard library JAR. - `target/DVUploader-v1.4.0.jar`: Executable "fat" JAR containing all dependencies. +Testing: + +Basic functionality tests for Dataverse can be run using Maven. These tests require a live Dataverse instance and valid credentials. + +1. Copy `test.properties.example` to `test.properties`. +2. Edit `test.properties` and provide your Dataverse server URL, API key, and a test Dataset PID (DOI). +3. Run the tests: +```bash +mvn test +``` + +Alternatively, you can provide configuration via system properties or environment variables: +```bash +mvn test -Ddataverse.server=... -Ddataverse.api_key=... -Ddataverse.dataset_pid=... +``` + Usage: See wiki: https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse diff --git a/pom.xml b/pom.xml index 5929de1..ef7de52 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,11 @@ ${maven.compiler.target} + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + org.apache.maven.plugins maven-jar-plugin @@ -123,5 +128,17 @@ jakarta.json ${jakarta.json.version} + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.10.2 + test + diff --git a/src/main/java/org/sead/uploader/AbstractUploader.java b/src/main/java/org/sead/uploader/AbstractUploader.java index bfc71a5..a44800d 100644 --- a/src/main/java/org/sead/uploader/AbstractUploader.java +++ b/src/main/java/org/sead/uploader/AbstractUploader.java @@ -74,6 +74,15 @@ public abstract class AbstractUploader { protected Set excluded = new HashSet(); protected static List requests = new ArrayList(); + public void clearRequests() { + requests.clear(); + clearCache(); + } + + public void clearCache() { + // To be overridden by subclasses to clear internal caches + } + protected static String server = null; PrintWriter pw = null; diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index 3b7a471..73a4842 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -318,8 +318,14 @@ public CloseableHttpClient getSharedHttpClient() { return httpclient; } + @Override + public void clearCache() { + datasetMDRetrieved = false; + existingItems = null; + } + + private boolean datasetMDRetrieved = false; HashMap existingItems = null; - boolean datasetMDRetrieved = false; CloseableHttpClient httpclient = null; @@ -1000,8 +1006,8 @@ private String multipartDirectFileUpload(Resource file, String path, int retries MessageDigest messageDigest = MessageDigest.getInstance(fixityAlgorithm); try (InputStream inStream = file.getInputStream(); DigestInputStream digestInputStream = new DigestInputStream(inStream, messageDigest)) { - // This is hte new form for requests - keeping the example but won't update until we can change all - //HttpUriRequest httpput = RequestBuilder.put() + // This is the new form for requests - keeping the example but won't update until we can change all + // HttpUriRequest httpput = RequestBuilder.put() // .setUri(uploadUrl) // .setHeader("x-amz-tagging", "dv-state=temp") // .setEntity(new InputStreamEntity(digestInputStream, file.length())) diff --git a/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java new file mode 100644 index 0000000..0ef9d3e --- /dev/null +++ b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java @@ -0,0 +1,198 @@ +package org.sead.uploader.dataverse; + +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.*; +import org.sead.uploader.util.UploaderException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Basic tests for DVUploader. + * These tests require a live Dataverse instance and valid credentials. + * Configure via test.properties file or environment variables. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class DVUploaderTest { + + private String server; + private String apiKey; + private String datasetPID; + private DVUploader uploader; + + @BeforeAll + public void setup() throws IOException { + Properties props = new Properties(); + File propFile = new File("test.properties"); + if (propFile.exists()) { + try (FileInputStream fis = new FileInputStream(propFile)) { + props.load(fis); + } + } + + server = System.getProperty("dataverse.server", props.getProperty("dataverse.server", System.getenv("DATAVERSE_SERVER"))); + apiKey = System.getProperty("dataverse.api_key", props.getProperty("dataverse.api_key", System.getenv("DATAVERSE_API_KEY"))); + datasetPID = System.getProperty("dataverse.dataset_pid", props.getProperty("dataverse.dataset_pid", System.getenv("DATAVERSE_DATASET_PID"))); + + // Skip tests if configuration is missing + Assumptions.assumeTrue(server != null && apiKey != null && datasetPID != null, + "Test configuration missing. Provide dataverse.server, dataverse.api_key, and dataverse.dataset_pid " + + "via test.properties, system properties, or environment variables."); + + uploader = new DVUploader(); + DVUploader.setUploader(uploader); + + // Initial setup of credentials + uploader.parseArgs(new String[]{"-server=" + server, "-key=" + apiKey, "-did=" + datasetPID}); + } + + @BeforeEach + public void resetUploader() { + uploader.clearRequests(); + // Re-apply common args to ensure they are set + uploader.parseArgs(new String[]{"-server=" + server, "-key=" + apiKey, "-did=" + datasetPID}); + } + + @AfterAll + public void cleanup() throws IOException { + if (uploader == null) return; + + System.out.println("Cleaning up uploaded files..."); + CloseableHttpClient httpClient = uploader.getSharedHttpClient(); + + // Get all files in the dataset to find their IDs + String url = server + "/api/datasets/:persistentId/versions/:latest/files?key=" + apiKey + "&persistentId=" + datasetPID; + HttpGet get = new HttpGet(url); + + try (CloseableHttpResponse response = httpClient.execute(get, uploader.getLocalContext())) { + if (response.getStatusLine().getStatusCode() == 200) { + String res = EntityUtils.toString(response.getEntity()); + JSONArray data = new JSONObject(res).getJSONArray("data"); + for (int i = 0; i < data.length(); i++) { + JSONObject fileEntry = data.getJSONObject(i); + JSONObject dataFile = fileEntry.getJSONObject("dataFile"); + String filename = dataFile.getString("filename"); + + // Cleanup any files starting with our test prefixes + if (filename.startsWith("dvuploader-test") || filename.startsWith("dvuploader-large")) { + long id = dataFile.getLong("id"); + deleteFile(httpClient, id); + } + } + } + } + } + + private void deleteFile(CloseableHttpClient httpClient, long id) throws IOException { + String url = server + "/api/files/" + id + "?key=" + apiKey; + HttpDelete delete = new HttpDelete(url); + try (CloseableHttpResponse response = httpClient.execute(delete, uploader.getLocalContext())) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 204 || statusCode == 200) { + System.out.println("Deleted file ID: " + id + " (Status: " + statusCode + ")"); + } else { + System.err.println("Failed to delete file ID: " + id + " Status: " + statusCode); + } + EntityUtils.consumeQuietly(response.getEntity()); + } + } + + @Test + public void testSimpleUpload() throws IOException, UploaderException { + Path tempFile = Files.createTempFile("dvuploader-test", ".txt"); + String filename = tempFile.getFileName().toString(); + Files.writeString(tempFile, "Hello Dataverse!"); + + try { + uploader.parseArgs(new String[]{tempFile.toAbsolutePath().toString()}); + uploader.processRequests(); + + assertTrue(isFileInDataset(filename), "File " + filename + " should be in dataset after upload"); + } finally { + Files.deleteIfExists(tempFile); + } + } + + @Test + public void testDuplicateUpload() throws IOException, UploaderException { + Path tempFile = Files.createTempFile("dvuploader-test-dup", ".txt"); + String filename = tempFile.getFileName().toString(); + Files.writeString(tempFile, "Duplicate Content"); + + try { + // First upload + uploader.parseArgs(new String[]{tempFile.toAbsolutePath().toString()}); + uploader.processRequests(); + assertTrue(isFileInDataset(filename), "File " + filename + " should be in dataset after first upload"); + + // Clear requests and re-add the same file + uploader.clearRequests(); + uploader.parseArgs(new String[]{tempFile.toAbsolutePath().toString()}); + + // Second upload - should see it exists and not re-upload (this is handled internally by DVUploader) + uploader.processRequests(); + + // Verify it's still there and there's only one (by name) + assertEquals(1, countFileInDataset(filename), "There should be exactly one file named " + filename + " in the dataset"); + } finally { + Files.deleteIfExists(tempFile); + } + } + + @Test + public void testLargeFileUpload() throws IOException, UploaderException { + // Create a > 5MB file to trigger multipart upload + Path largeFile = Files.createTempFile("dvuploader-large", ".bin"); + String filename = largeFile.getFileName().toString(); + byte[] data = new byte[6 * 1024 * 1024]; // 6 MB + Files.write(largeFile, data); + + try { + uploader.parseArgs(new String[]{largeFile.toAbsolutePath().toString()}); + uploader.processRequests(); + assertTrue(isFileInDataset(filename), "Large file " + filename + " should be in dataset after upload"); + } finally { + Files.deleteIfExists(largeFile); + } + } + + private boolean isFileInDataset(String filename) throws IOException { + return countFileInDataset(filename) > 0; + } + + private int countFileInDataset(String filename) throws IOException { + CloseableHttpClient httpClient = uploader.getSharedHttpClient(); + String url = server + "/api/datasets/:persistentId/versions/:latest/files?key=" + apiKey + "&persistentId=" + datasetPID; + HttpGet get = new HttpGet(url); + int count = 0; + try (CloseableHttpResponse response = httpClient.execute(get, uploader.getLocalContext())) { + if (response.getStatusLine().getStatusCode() == 200) { + String res = EntityUtils.toString(response.getEntity()); + JSONArray data = new JSONObject(res).getJSONArray("data"); + for (int i = 0; i < data.length(); i++) { + JSONObject fileEntry = data.getJSONObject(i); + if (fileEntry.getJSONObject("dataFile").getString("filename").equals(filename)) { + count++; + } + } + } + EntityUtils.consumeQuietly(response.getEntity()); + } + return count; + } +} diff --git a/test.properties.example b/test.properties.example new file mode 100644 index 0000000..cf9bf3d --- /dev/null +++ b/test.properties.example @@ -0,0 +1,5 @@ +# Dataverse Test Configuration +# Copy this file to test.properties and fill in the values +dataverse.server=https://demo.dataverse.org +dataverse.api_key=YOUR_API_KEY +dataverse.dataset_pid=doi:10.5072/FK2/XXXXXX From 7830e3baf057268056beaee20271d4bacb86a9a5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 23 Jul 2026 14:03:41 -0400 Subject: [PATCH 3/7] mask API key --- src/main/java/org/sead/uploader/AbstractUploader.java | 6 +++++- src/main/java/org/sead/uploader/dataverse/DVUploader.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/sead/uploader/AbstractUploader.java b/src/main/java/org/sead/uploader/AbstractUploader.java index a44800d..f9c1a22 100644 --- a/src/main/java/org/sead/uploader/AbstractUploader.java +++ b/src/main/java/org/sead/uploader/AbstractUploader.java @@ -126,7 +126,11 @@ public static void printStatus(float s) { public void parseArgs(String[] args) { for (String arg : args) { - // println("Arg is : " + arg); + if (arg.startsWith("-key" + argSeparator)) { + println("Arg is : -key" + argSeparator + "MASKED"); + } else { + println("Arg is : " + arg); + } if (arg.equalsIgnoreCase("-listonly")) { listonly = true; println("List Only Mode"); diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index 73a4842..6ca9b33 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -186,7 +186,7 @@ public boolean parseCustomArg(String arg) { if (arg.startsWith("-key")) { apiKey = arg.substring(arg.indexOf(argSeparator) + 1); - println("Using apiKey: " + apiKey); + println("Using apiKey: MASKED"); return true; } else if (arg.startsWith("-did")) { datasetPID = arg.substring(arg.indexOf(argSeparator) + 1); From 29aadba7c84802ef3d5b2878436727a83882bd3b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 23 Jul 2026 14:59:03 -0400 Subject: [PATCH 4/7] test fixes, add part_size param --- .../org/sead/uploader/AbstractUploader.java | 15 +++ .../sead/uploader/dataverse/DVUploader.java | 15 +++ .../uploader/dataverse/DVUploaderTest.java | 93 ++++++++++++++++++- 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/sead/uploader/AbstractUploader.java b/src/main/java/org/sead/uploader/AbstractUploader.java index f9c1a22..44edbd4 100644 --- a/src/main/java/org/sead/uploader/AbstractUploader.java +++ b/src/main/java/org/sead/uploader/AbstractUploader.java @@ -76,6 +76,21 @@ public abstract class AbstractUploader { public void clearRequests() { requests.clear(); + max = Long.MAX_VALUE; + skip = 0l; + globalFileCount = 0l; + totalBytes = 0l; + listonly = false; + verify = false; + importRO = false; + merge = true; + excluded.clear(); + hashIssues.clear(); + roDataIdToNewId.clear(); + roCollIdToNewId.clear(); + roFolderProxy.clear(); + server = null; + bagLocation = null; clearCache(); } diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index 6ca9b33..a55ce63 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -322,6 +322,21 @@ public CloseableHttpClient getSharedHttpClient() { public void clearCache() { datasetMDRetrieved = false; existingItems = null; + hashIssues.clear(); + apiKey = null; + datasetPID = null; + alias = null; + oldServer = false; + maxWaitTime = 60; + recurse = false; + directUpload = true; + trustCerts = false; + singleFile = false; + noIngest = false; + fixNames = true; + httpclient = null; + cm = null; + fixityAlgorithm = "MD5"; } private boolean datasetMDRetrieved = false; diff --git a/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java index 0ef9d3e..93968b7 100644 --- a/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java +++ b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java @@ -14,11 +14,13 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; @@ -33,6 +35,7 @@ public class DVUploaderTest { private String server; private String apiKey; private String datasetPID; + private long partSize; private DVUploader uploader; @BeforeAll @@ -48,6 +51,13 @@ public void setup() throws IOException { server = System.getProperty("dataverse.server", props.getProperty("dataverse.server", System.getenv("DATAVERSE_SERVER"))); apiKey = System.getProperty("dataverse.api_key", props.getProperty("dataverse.api_key", System.getenv("DATAVERSE_API_KEY"))); datasetPID = System.getProperty("dataverse.dataset_pid", props.getProperty("dataverse.dataset_pid", System.getenv("DATAVERSE_DATASET_PID"))); + + String partSizeStr = System.getProperty("dataverse.part_size", props.getProperty("dataverse.part_size", System.getenv("DATAVERSE_PART_SIZE"))); + if (partSizeStr != null) { + partSize = Long.parseLong(partSizeStr); + } else { + partSize = 5 * 1024 * 1024; // Default 5MB + } // Skip tests if configuration is missing Assumptions.assumeTrue(server != null && apiKey != null && datasetPID != null, @@ -156,11 +166,23 @@ public void testDuplicateUpload() throws IOException, UploaderException { @Test public void testLargeFileUpload() throws IOException, UploaderException { - // Create a > 5MB file to trigger multipart upload + System.out.println("Using part size: " + partSize); + // Create a file slightly larger than partSize to trigger multipart upload + long fileSize = partSize + (1024 * 1024); // partSize + 1 MB + Path largeFile = Files.createTempFile("dvuploader-large", ".bin"); String filename = largeFile.getFileName().toString(); - byte[] data = new byte[6 * 1024 * 1024]; // 6 MB - Files.write(largeFile, data); + + System.out.println("Creating " + fileSize + " bytes temp file: " + filename); + try (OutputStream os = Files.newOutputStream(largeFile)) { + byte[] buffer = new byte[1024 * 1024]; // 1MB buffer + long written = 0; + while (written < fileSize) { + int toWrite = (int) Math.min(buffer.length, fileSize - written); + os.write(buffer, 0, toWrite); + written += toWrite; + } + } try { uploader.parseArgs(new String[]{largeFile.toAbsolutePath().toString()}); @@ -171,6 +193,71 @@ public void testLargeFileUpload() throws IOException, UploaderException { } } + @Test + public void testDirectoryTreeUploadWithLimit() throws IOException, UploaderException { + Path tempDir = Files.createTempDirectory("dvuploader-test-tree"); + try { + Path file1 = Files.createFile(tempDir.resolve("dvuploader-test-tree1.txt")); + Files.writeString(file1, "File 1 content"); + Path subDir = Files.createDirectory(tempDir.resolve("subdir")); + Path file2 = Files.createFile(subDir.resolve("dvuploader-test-tree2.txt")); + Files.writeString(file2, "File 2 content"); + Path file3 = Files.createFile(subDir.resolve("dvuploader-test-tree3.txt")); + Files.writeString(file3, "File 3 content"); + + String[] filenames = { + file1.getFileName().toString(), + file2.getFileName().toString(), + file3.getFileName().toString() + }; + + // Run 1: limit = 1, recurse + System.out.println("Run 1: limit=1"); + uploader.parseArgs(new String[]{"-limit=1", "-recurse", tempDir.toAbsolutePath().toString()}); + uploader.processRequests(); + + int count = countFilesFromSet(filenames); + assertEquals(1, count, "Should have exactly 1 file uploaded in first run"); + + // Run 2: no limit, recurse + System.out.println("Run 2: full upload"); + uploader.clearRequests(); + uploader.parseArgs(new String[]{"-server=" + server, "-key=" + apiKey, "-did=" + datasetPID}); // Re-add common args + uploader.parseArgs(new String[]{"-recurse", tempDir.toAbsolutePath().toString()}); + uploader.processRequests(); + + count = countFilesFromSet(filenames); + assertEquals(3, count, "Should have all 3 files uploaded after second run"); + + } finally { + deleteDirectory(tempDir); + } + } + + + private int countFilesFromSet(String[] filenames) throws IOException { + int count = 0; + for (String f : filenames) { + if (isFileInDataset(f)) count++; + } + return count; + } + + private void deleteDirectory(Path path) throws IOException { + if (Files.isDirectory(path)) { + try (Stream entries = Files.list(path)) { + entries.forEach(p -> { + try { + deleteDirectory(p); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + } + Files.deleteIfExists(path); + } + private boolean isFileInDataset(String filename) throws IOException { return countFileInDataset(filename) > 0; } From 68063cc321ab2ba9e1d58d32c4a97e288f5f78f8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 23 Jul 2026 14:59:03 -0400 Subject: [PATCH 5/7] test fixes, add part_size param --- README.md | 2 +- .../org/sead/uploader/AbstractUploader.java | 15 +++ .../sead/uploader/dataverse/DVUploader.java | 15 +++ .../uploader/dataverse/DVUploaderTest.java | 93 ++++++++++++++++++- test.properties.example | 2 + 5 files changed, 123 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e9d630f..f728cad 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Testing: Basic functionality tests for Dataverse can be run using Maven. These tests require a live Dataverse instance and valid credentials. 1. Copy `test.properties.example` to `test.properties`. -2. Edit `test.properties` and provide your Dataverse server URL, API key, and a test Dataset PID (DOI). +2. Edit `test.properties` and provide your Dataverse server URL, API key, a test Dataset PID (DOI), and the part size for multipart uploads. 3. Run the tests: ```bash mvn test diff --git a/src/main/java/org/sead/uploader/AbstractUploader.java b/src/main/java/org/sead/uploader/AbstractUploader.java index f9c1a22..44edbd4 100644 --- a/src/main/java/org/sead/uploader/AbstractUploader.java +++ b/src/main/java/org/sead/uploader/AbstractUploader.java @@ -76,6 +76,21 @@ public abstract class AbstractUploader { public void clearRequests() { requests.clear(); + max = Long.MAX_VALUE; + skip = 0l; + globalFileCount = 0l; + totalBytes = 0l; + listonly = false; + verify = false; + importRO = false; + merge = true; + excluded.clear(); + hashIssues.clear(); + roDataIdToNewId.clear(); + roCollIdToNewId.clear(); + roFolderProxy.clear(); + server = null; + bagLocation = null; clearCache(); } diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index 6ca9b33..a55ce63 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -322,6 +322,21 @@ public CloseableHttpClient getSharedHttpClient() { public void clearCache() { datasetMDRetrieved = false; existingItems = null; + hashIssues.clear(); + apiKey = null; + datasetPID = null; + alias = null; + oldServer = false; + maxWaitTime = 60; + recurse = false; + directUpload = true; + trustCerts = false; + singleFile = false; + noIngest = false; + fixNames = true; + httpclient = null; + cm = null; + fixityAlgorithm = "MD5"; } private boolean datasetMDRetrieved = false; diff --git a/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java index 0ef9d3e..93968b7 100644 --- a/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java +++ b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java @@ -14,11 +14,13 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; @@ -33,6 +35,7 @@ public class DVUploaderTest { private String server; private String apiKey; private String datasetPID; + private long partSize; private DVUploader uploader; @BeforeAll @@ -48,6 +51,13 @@ public void setup() throws IOException { server = System.getProperty("dataverse.server", props.getProperty("dataverse.server", System.getenv("DATAVERSE_SERVER"))); apiKey = System.getProperty("dataverse.api_key", props.getProperty("dataverse.api_key", System.getenv("DATAVERSE_API_KEY"))); datasetPID = System.getProperty("dataverse.dataset_pid", props.getProperty("dataverse.dataset_pid", System.getenv("DATAVERSE_DATASET_PID"))); + + String partSizeStr = System.getProperty("dataverse.part_size", props.getProperty("dataverse.part_size", System.getenv("DATAVERSE_PART_SIZE"))); + if (partSizeStr != null) { + partSize = Long.parseLong(partSizeStr); + } else { + partSize = 5 * 1024 * 1024; // Default 5MB + } // Skip tests if configuration is missing Assumptions.assumeTrue(server != null && apiKey != null && datasetPID != null, @@ -156,11 +166,23 @@ public void testDuplicateUpload() throws IOException, UploaderException { @Test public void testLargeFileUpload() throws IOException, UploaderException { - // Create a > 5MB file to trigger multipart upload + System.out.println("Using part size: " + partSize); + // Create a file slightly larger than partSize to trigger multipart upload + long fileSize = partSize + (1024 * 1024); // partSize + 1 MB + Path largeFile = Files.createTempFile("dvuploader-large", ".bin"); String filename = largeFile.getFileName().toString(); - byte[] data = new byte[6 * 1024 * 1024]; // 6 MB - Files.write(largeFile, data); + + System.out.println("Creating " + fileSize + " bytes temp file: " + filename); + try (OutputStream os = Files.newOutputStream(largeFile)) { + byte[] buffer = new byte[1024 * 1024]; // 1MB buffer + long written = 0; + while (written < fileSize) { + int toWrite = (int) Math.min(buffer.length, fileSize - written); + os.write(buffer, 0, toWrite); + written += toWrite; + } + } try { uploader.parseArgs(new String[]{largeFile.toAbsolutePath().toString()}); @@ -171,6 +193,71 @@ public void testLargeFileUpload() throws IOException, UploaderException { } } + @Test + public void testDirectoryTreeUploadWithLimit() throws IOException, UploaderException { + Path tempDir = Files.createTempDirectory("dvuploader-test-tree"); + try { + Path file1 = Files.createFile(tempDir.resolve("dvuploader-test-tree1.txt")); + Files.writeString(file1, "File 1 content"); + Path subDir = Files.createDirectory(tempDir.resolve("subdir")); + Path file2 = Files.createFile(subDir.resolve("dvuploader-test-tree2.txt")); + Files.writeString(file2, "File 2 content"); + Path file3 = Files.createFile(subDir.resolve("dvuploader-test-tree3.txt")); + Files.writeString(file3, "File 3 content"); + + String[] filenames = { + file1.getFileName().toString(), + file2.getFileName().toString(), + file3.getFileName().toString() + }; + + // Run 1: limit = 1, recurse + System.out.println("Run 1: limit=1"); + uploader.parseArgs(new String[]{"-limit=1", "-recurse", tempDir.toAbsolutePath().toString()}); + uploader.processRequests(); + + int count = countFilesFromSet(filenames); + assertEquals(1, count, "Should have exactly 1 file uploaded in first run"); + + // Run 2: no limit, recurse + System.out.println("Run 2: full upload"); + uploader.clearRequests(); + uploader.parseArgs(new String[]{"-server=" + server, "-key=" + apiKey, "-did=" + datasetPID}); // Re-add common args + uploader.parseArgs(new String[]{"-recurse", tempDir.toAbsolutePath().toString()}); + uploader.processRequests(); + + count = countFilesFromSet(filenames); + assertEquals(3, count, "Should have all 3 files uploaded after second run"); + + } finally { + deleteDirectory(tempDir); + } + } + + + private int countFilesFromSet(String[] filenames) throws IOException { + int count = 0; + for (String f : filenames) { + if (isFileInDataset(f)) count++; + } + return count; + } + + private void deleteDirectory(Path path) throws IOException { + if (Files.isDirectory(path)) { + try (Stream entries = Files.list(path)) { + entries.forEach(p -> { + try { + deleteDirectory(p); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + } + Files.deleteIfExists(path); + } + private boolean isFileInDataset(String filename) throws IOException { return countFileInDataset(filename) > 0; } diff --git a/test.properties.example b/test.properties.example index cf9bf3d..705a9ea 100644 --- a/test.properties.example +++ b/test.properties.example @@ -3,3 +3,5 @@ dataverse.server=https://demo.dataverse.org dataverse.api_key=YOUR_API_KEY dataverse.dataset_pid=doi:10.5072/FK2/XXXXXX +# Part size for multipart upload in bytes (e.g. 5242880 for 5MB) +dataverse.part_size=5242880 From 004fe147b15b7fcdd0d7acba994129d00f1ecb0b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 24 Jul 2026 12:00:15 -0400 Subject: [PATCH 6/7] remove redundant call --- .../org/sead/uploader/AbstractUploader.java | 2 - .../sead/uploader/dataverse/RetryTest.java | 162 ++++++++++++++++++ 2 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/sead/uploader/dataverse/RetryTest.java diff --git a/src/main/java/org/sead/uploader/AbstractUploader.java b/src/main/java/org/sead/uploader/AbstractUploader.java index 44edbd4..b09cd46 100644 --- a/src/main/java/org/sead/uploader/AbstractUploader.java +++ b/src/main/java/org/sead/uploader/AbstractUploader.java @@ -234,8 +234,6 @@ public void processRequests() { // the // collection postProcessCollection(); - } else { - postProcessChildren(file); } } else { newUri = null; // listonly - report no changes diff --git a/src/test/java/org/sead/uploader/dataverse/RetryTest.java b/src/test/java/org/sead/uploader/dataverse/RetryTest.java new file mode 100644 index 0000000..cd41251 --- /dev/null +++ b/src/test/java/org/sead/uploader/dataverse/RetryTest.java @@ -0,0 +1,162 @@ +package org.sead.uploader.dataverse; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RetryTest { + + private HttpServer server; + private int port; + private CloseableHttpClient httpClient; + + @BeforeEach + public void setup() throws IOException, NoSuchFieldException, IllegalAccessException { + server = HttpServer.create(new InetSocketAddress(0), 0); + server.setExecutor(null); + server.start(); + port = server.getAddress().getPort(); + httpClient = HttpClients.createDefault(); + + // Speed up tests by reducing retry delays + setStaticField(DVUploader.class, "uploadUrlBaseRetryDelayMs", 10); + setStaticField(DVUploader.class, "uploadUrlMaxRetryDelayMs", 100); + } + + private void setStaticField(Class clazz, String fieldName, Object value) throws NoSuchFieldException, IllegalAccessException { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(null, value); + } + + private Object getStaticField(Class clazz, String fieldName) throws NoSuchFieldException, IllegalAccessException { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(null); + } + + @AfterEach + public void teardown() throws IOException { + if (server != null) { + server.stop(0); + } + if (httpClient != null) { + httpClient.close(); + } + } + + @Test + public void test429Retry() throws IOException { + AtomicInteger callCount = new AtomicInteger(0); + server.createContext("/api/datasets/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + int count = callCount.incrementAndGet(); + if (count == 1) { + exchange.sendResponseHeaders(429, -1); + } else { + byte[] response = "OK".getBytes(); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + } + }); + + HttpGet request = new HttpGet("http://localhost:" + port + "/api/datasets/test"); + try (CloseableHttpResponse response = DVUploader.executeWithRetry(request, httpClient, null)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(2, callCount.get()); + } + } + + @Test + public void test50xRetry() throws IOException { + AtomicInteger callCount = new AtomicInteger(0); + server.createContext("/s3/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + int count = callCount.incrementAndGet(); + if (count <= 2) { + exchange.sendResponseHeaders(503, -1); + } else { + byte[] response = "OK".getBytes(); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + } + }); + + HttpGet request = new HttpGet("http://localhost:" + port + "/s3/test"); + try (CloseableHttpResponse response = DVUploader.executeWithRetry(request, httpClient, null)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(3, callCount.get()); + } + } + + @Test + public void testGlobalSlowdown() throws IOException { + // Reset state + // Since fields are private, I can't reset them easily unless I make them package-private or use reflection. + // But I can just check that it increases. + + server.createContext("/api/datasets/slowdown", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + exchange.sendResponseHeaders(429, -1); + } + }); + + HttpGet request = new HttpGet("http://localhost:" + port + "/api/datasets/slowdown"); + // We expect it to retry 5 times and then return 429 + try (CloseableHttpResponse response = DVUploader.executeWithRetry(request, httpClient, null)) { + assertEquals(429, response.getStatusLine().getStatusCode()); + } + + // Now try another request, it should wait at least 5 * 50ms = 250ms more than before if we hit 429 5 times. + // Actually, uploadUrlInterRequestDelayMs is increased by 50ms on each 429. + } + + @Test + public void testClearCacheResetsRetryConfigButNotTiming() throws IOException, NoSuchFieldException, IllegalAccessException { + // 1. Manually set some non-default values + setStaticField(DVUploader.class, "uploadUrlMaxRetries", 10); + setStaticField(DVUploader.class, "uploadUrlBaseRetryDelayMs", 5000); + setStaticField(DVUploader.class, "uploadUrlMaxRetryDelayMs", 120000); + + setStaticField(DVUploader.class, "uploadUrlCooldownUntil", 123456789L); + setStaticField(DVUploader.class, "uploadUrlInterRequestDelayMs", 500); + setStaticField(DVUploader.class, "lastUploadUrlRequestTimestamp", 987654321L); + + // 2. Call clearCache + new DVUploader().clearCache(); + + // 3. Verify config is reset + assertEquals(5, getStaticField(DVUploader.class, "uploadUrlMaxRetries"), "uploadUrlMaxRetries should be reset to default"); + assertEquals(2000, getStaticField(DVUploader.class, "uploadUrlBaseRetryDelayMs"), "uploadUrlBaseRetryDelayMs should be reset to default"); + assertEquals(60000, getStaticField(DVUploader.class, "uploadUrlMaxRetryDelayMs"), "uploadUrlMaxRetryDelayMs should be reset to default"); + + // 4. Verify timing state is NOT reset + assertEquals(123456789L, getStaticField(DVUploader.class, "uploadUrlCooldownUntil"), "uploadUrlCooldownUntil should NOT be reset"); + assertEquals(500, getStaticField(DVUploader.class, "uploadUrlInterRequestDelayMs"), "uploadUrlInterRequestDelayMs should NOT be reset"); + assertEquals(987654321L, getStaticField(DVUploader.class, "lastUploadUrlRequestTimestamp"), "lastUploadUrlRequestTimestamp should NOT be reset"); + } +} From 74aacc30a6fbf35203253d1d5d1b9680d2a2dfde Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 24 Jul 2026 12:01:05 -0400 Subject: [PATCH 7/7] add S3 retries, 429 DV retries (test was in last commit by mistake) --- .../sead/uploader/dataverse/DVUploader.java | 154 ++++++++++++++++-- .../uploader/dataverse/HttpPartUploadJob.java | 34 ++-- 2 files changed, 162 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index a55ce63..11ae3d5 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -50,6 +50,7 @@ import org.apache.commons.codec.binary.Hex; import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; @@ -108,6 +109,13 @@ public class DVUploader extends AbstractUploader { private int timeout = 1200; private int httpConcurrency = 4; + private static int uploadUrlMaxRetries = 5; + private static int uploadUrlBaseRetryDelayMs = 2000; + private static int uploadUrlMaxRetryDelayMs = 60000; + private static long uploadUrlCooldownUntil = 0; + private static int uploadUrlInterRequestDelayMs = 0; + private static long lastUploadUrlRequestTimestamp = 0; + //private static long mpSizeLimit = 5 * 1024 * 1024; private RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) @@ -254,7 +262,7 @@ public void processRequests() { String serviceUrl = server + "/api/files/fixityAlgorithm"; HttpGet httpget = new HttpGet(serviceUrl); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); try { switch (response.getStatusLine().getStatusCode()) { case 200: @@ -302,6 +310,107 @@ public HttpClientContext authenticate() { return new HttpClientContext(); } + private static synchronized void updateUploadUrlCooldown(long delayMs) { + uploadUrlCooldownUntil = Math.max(uploadUrlCooldownUntil, System.currentTimeMillis() + delayMs); + } + + private static synchronized void recordUploadUrlRequest() { + lastUploadUrlRequestTimestamp = System.currentTimeMillis(); + } + + private static synchronized void waitForUploadUrlCooldown() { + long now = System.currentTimeMillis(); + long waitTime = Math.max(uploadUrlCooldownUntil - now, (lastUploadUrlRequestTimestamp + uploadUrlInterRequestDelayMs) - now); + if (waitTime > 0) { + try { + // println("Waiting for cooldown: " + waitTime + "ms"); + Thread.sleep(waitTime); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + private static long getRetryAfterDelayMs(CloseableHttpResponse response) { + org.apache.http.Header header = response.getFirstHeader("Retry-After"); + if (header != null) { + try { + // Can be a number of seconds or an HTTP-date + String value = header.getValue(); + if (value.matches("\\d+")) { + return Long.parseLong(value) * 1000; + } + // Handle HTTP-date if necessary, but most APIs use seconds + } catch (Exception e) { + // Ignore + } + } + return 0; + } + + public static CloseableHttpResponse executeWithRetry(org.apache.http.client.methods.HttpUriRequest request, CloseableHttpClient client, HttpClientContext context) throws IOException { + int retryCount = 0; + while (true) { + boolean isDatasetApi = request.getURI().getPath().contains("/api/datasets"); + if (isDatasetApi) { + waitForUploadUrlCooldown(); + recordUploadUrlRequest(); + } + + CloseableHttpResponse response = client.execute(request, context); + int status = response.getStatusLine().getStatusCode(); + + if (isDatasetApi && status == 429 && retryCount < uploadUrlMaxRetries) { + long retryAfterDelayMs = getRetryAfterDelayMs(response); + long recoveryDelayMs = Math.max( + retryAfterDelayMs, + Math.min(uploadUrlBaseRetryDelayMs * (long) Math.pow(2, retryCount), (long) uploadUrlMaxRetryDelayMs) + ); + + synchronized (DVUploader.class) { + uploadUrlInterRequestDelayMs += 50; + } + updateUploadUrlCooldown(recoveryDelayMs); + + EntityUtils.consumeQuietly(response.getEntity()); + response.close(); + + println("Retrying call to " + request.getURI() + " due to 429 in " + recoveryDelayMs + "ms (attempt " + (retryCount + 1) + " of " + uploadUrlMaxRetries + ")"); + try { + Thread.sleep(recoveryDelayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during retry wait", e); + } + retryCount++; + continue; + } + + if (!isDatasetApi && status >= 500 && status <= 599 && retryCount < 3) { + HttpEntity entity = (request instanceof HttpEntityEnclosingRequest) ? ((HttpEntityEnclosingRequest) request).getEntity() : null; + if (entity == null || entity.isRepeatable()) { + long baseDelay = 100; + long delay = retryCount == 0 ? baseDelay : baseDelay * (long) Math.pow(2, retryCount); + + EntityUtils.consumeQuietly(response.getEntity()); + response.close(); + + println("Retrying call to " + request.getURI() + " due to " + status + " in " + delay + "ms (attempt " + (retryCount + 1) + " of 3)"); + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during retry wait", e); + } + retryCount++; + continue; + } + } + + return response; + } + } + public CloseableHttpClient getSharedHttpClient() { if (httpclient == null) { try { @@ -337,6 +446,18 @@ public void clearCache() { httpclient = null; cm = null; fixityAlgorithm = "MD5"; + + // Reset retry configuration to defaults + // These can't be changed via command line currently but are setable if DVUploader is used as a library (as in tests) + uploadUrlMaxRetries = 5; + uploadUrlBaseRetryDelayMs = 2000; + uploadUrlMaxRetryDelayMs = 60000; + + // Note: We intentionally do NOT reset uploadUrlCooldownUntil, + // uploadUrlInterRequestDelayMs, and lastUploadUrlRequestTimestamp here. + // These values represent the current rate-limiting state of the Dataverse + // server and should persist across cache clears (e.g., between tests) + // to avoid hitting rate limits repeatedly in a short time window. } private boolean datasetMDRetrieved = false; @@ -386,7 +507,7 @@ public String itemExists(String path, Resource item) { + "&persistentId=" + datasetPID; HttpGet httpget = new HttpGet(serviceUrl); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); JSONArray datafileList = null; try { switch (response.getStatusLine().getStatusCode()) { @@ -558,7 +679,7 @@ private String createDataset(Resource dir, String path) { httppost.setEntity(se); httppost.addHeader("Content-Type","application/json-ld"); - CloseableHttpResponse response = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httppost, httpclient, getLocalContext()); try { if (response.getStatusLine().getStatusCode() == 201) { HttpEntity resEntity = response.getEntity(); @@ -631,7 +752,7 @@ protected void postProcessChildren(Resource dir) { HttpEntity reqEntity = meb.build(); httppost.setEntity(reqEntity); try { - CloseableHttpResponse postResponse = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse postResponse = executeWithRetry(httppost, httpclient, getLocalContext()); int postStatus = postResponse.getStatusLine().getStatusCode(); String postRes = null; @@ -651,6 +772,15 @@ protected void postProcessChildren(Resource dir) { if (fileResult.has("error Code: ")) { errArray.put(fileResult); errIds.add(fileResult.getString("storageIdentifier")); + } else { + // Successfully added - clear metadata to avoid re-registration + for (Resource r : dir.listResources()) { + if (!r.isDirectory() && r.getMetadata().has("storageIdentifier") && + r.getMetadata().getString("storageIdentifier").equals(fileResult.getString("storageIdentifier"))) { + r.setMetadata(new org.json.JSONObject()); + break; + } + } } } println((jsonData.length() - errIds.size()) + " files successfully added from this folder"); @@ -743,7 +873,7 @@ protected void postProcessCollection() { httppost.setEntity(body); - CloseableHttpResponse response = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httppost, httpclient, getLocalContext()); int status = response.getStatusLine().getStatusCode(); String res = null; @@ -858,7 +988,7 @@ protected String uploadDatafile(Resource file, String path) { HttpEntity reqEntity = meb.build(); httppost.setEntity(reqEntity); - CloseableHttpResponse response = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httppost, httpclient, getLocalContext()); try { int status = response.getStatusLine().getStatusCode(); String res = null; @@ -930,7 +1060,7 @@ private boolean isLocked() { urlString = urlString + "?persistentId=" + datasetPID + "&key=" + apiKey; HttpGet httpget = new HttpGet(urlString); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); try { if (response.getStatusLine().getStatusCode() == 200) { HttpEntity resEntity = response.getEntity(); @@ -989,7 +1119,7 @@ private String multipartDirectFileUpload(Resource file, String path, int retries String urlString = server + "/api/datasets/:persistentId/uploadurls"; urlString = urlString + "?persistentId=" + datasetPID + "&key=" + apiKey + "&size=" + file.length(); HttpGet httpget = new HttpGet(urlString); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); try { int status = response.getStatusLine().getStatusCode(); @@ -1028,7 +1158,7 @@ private String multipartDirectFileUpload(Resource file, String path, int retries // .setEntity(new InputStreamEntity(digestInputStream, file.length())) // .build(); httpput.setEntity(new InputStreamEntity(digestInputStream, file.length())); - CloseableHttpResponse putResponse = httpclient.execute(httpput); + CloseableHttpResponse putResponse = executeWithRetry(httpput, httpclient, getLocalContext()); try { int putStatus = putResponse.getStatusLine().getStatusCode(); String putRes = null; @@ -1165,7 +1295,7 @@ private String multipartDirectFileUpload(Resource file, String path, int retries completeUpload.setEntity(body); completeUpload.setHeader("Content-type", "application/json"); - response = httpclient.execute(completeUpload, getLocalContext()); + response = executeWithRetry(completeUpload, httpclient, getLocalContext()); EntityUtils.consumeQuietly(response.getEntity()); status = response.getStatusLine().getStatusCode(); if (status == 200) { @@ -1201,7 +1331,7 @@ private String multipartDirectFileUpload(Resource file, String path, int retries retries = 0; } else { HttpDelete delete = new HttpDelete(server + abortUrl + "&key=" + apiKey); - response = httpclient.execute(delete, getLocalContext()); + response = executeWithRetry(delete, httpclient, getLocalContext()); EntityUtils.consumeQuietly(response.getEntity()); status = response.getStatusLine().getStatusCode(); if (status != 204) { @@ -1278,7 +1408,7 @@ private String registerFileWithDataverse(Resource file, String path, String stor HttpEntity reqEntity = meb.build(); httppost.setEntity(reqEntity); try { - CloseableHttpResponse postResponse = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse postResponse = executeWithRetry(httppost, httpclient, getLocalContext()); int postStatus = postResponse.getStatusLine().getStatusCode(); String postRes = null; diff --git a/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java b/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java index 596e354..63cb862 100644 --- a/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java +++ b/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java @@ -63,17 +63,18 @@ public HttpPartUploadJob(int partNo, String url, Resource file, long size, Map e * @see java.lang.Runnable#run() */ public void run() { - int retries = 3; + int retryCount = 0; + int maxRetries = 3; //println("Starting upload of part: " + partNo); - while (retries > 0) { - if(retries <3) { - println("Retrying upload of part: " + partNo); + while (retryCount < maxRetries) { + if (retryCount > 0) { + println("Retrying upload of part: " + partNo + " (attempt " + (retryCount + 1) + " of " + maxRetries + ")"); } try (InputStream is = file.getInputStream((partNo - 1) * partSize, size)) { HttpPut httpput = new HttpPut(signedUrl); httpput.setEntity(new InputStreamEntity(is, size)); - CloseableHttpResponse putResponse = httpClient.execute(httpput); + CloseableHttpResponse putResponse = DVUploader.executeWithRetry(httpput, httpClient, localContext); int putStatus = putResponse.getStatusLine().getStatusCode(); String putRes = null; HttpEntity putEntity = putResponse.getEntity(); @@ -83,27 +84,32 @@ public void run() { if (putStatus == 200) { //Part successfully stored - parse the eTag from the response and it it to the Map String eTag = putResponse.getFirstHeader("ETag").getValue(); - eTag= eTag.replace("\"",""); + eTag = eTag.replace("\"", ""); eTags.put(Integer.toString(partNo), eTag); - retries = 0; - //println("Completed upload of part: " + partNo); + return; } else { - if (putStatus >= 500) { - println("Upload of part: " + partNo + " failed with status: " + putStatus + " (skipping)"); + if (putStatus >= 500 && putStatus <= 599) { + long baseDelay = 100; + long delay = retryCount == 0 ? baseDelay : baseDelay * (long) Math.pow(2, retryCount); + println("Upload of part: " + partNo + " failed with status: " + putStatus + ". Retrying in " + delay + "ms"); println("Error response: " + putResponse.getStatusLine() + " : " + putRes); - retries--; + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + retryCount++; } else { println("Upload of part: " + partNo + " failed with status: " + putStatus + " (retrying)"); println("Error response: " + putResponse.getStatusLine() + " : " + putRes); - - retries--; + retryCount++; } } } catch (IOException e) { e.printStackTrace(System.out); println("Error uploading part: " + partNo + " : " + e.getMessage()); - retries--; + retryCount++; } } }