diff --git a/.gitignore b/.gitignore index a6f89c2..bead6a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -/target/ \ No newline at end of file +/target/ +test.properties +DVUploaderLog_*.log \ No newline at end of file diff --git a/README.md b/README.md index 6242114..f728cad 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,30 @@ For Clowder/SEAD-specific information, see https://opensource.ncsa.illinois.edu/ Build: - mvn clean compile assembly:single - - Usage: See wiki: https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse +Run the following command to build the project and create the executable JAR: + +```bash +mvn clean package -DskipTests +``` + +This will produce: +- `target/DVUploader-1.4.0.jar`: Standard library JAR. +- `target/DVUploader-v1.4.0.jar`: Executable "fat" JAR containing all dependencies. + +Testing: + +Basic functionality tests for Dataverse can be run using Maven. These tests require a live Dataverse instance and valid credentials. + +1. Copy `test.properties.example` to `test.properties`. +2. Edit `test.properties` and provide your Dataverse server URL, API key, a test Dataset PID (DOI), and the part size for multipart uploads. +3. Run the tests: +```bash +mvn test +``` + +Alternatively, you can provide configuration via system properties or environment variables: +```bash +mvn test -Ddataverse.server=... -Ddataverse.api_key=... -Ddataverse.dataset_pid=... +``` + +Usage: See wiki: https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse diff --git a/pom.xml b/pom.xml index d3d2a69..ef7de52 100644 --- a/pom.xml +++ b/pom.xml @@ -4,25 +4,64 @@ 4.0.0 DVUploader DVUploader - 1.3.0-beta + 1.4.0 UTF-8 + 21 + 21 + 4.5.14 + 4.4.16 + 2.22.1 + 2.22 + 1.22.0 + 2.22.0 + 20260522 + 1.4.0 + 2.0.1 + 1.3.1 src/main/java maven-compiler-plugin - 3.11.0 + 3.13.0 - 1.8 - 1.8 + ${maven.compiler.source} + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.1 + + + + true + org.sead.uploader.dataverse.DVUploader + + org.apache.maven.plugins maven-assembly-plugin - 3.6.0 + 3.7.1 + + + make-assembly + package + + single + + + @@ -42,67 +81,64 @@ org.apache.httpcomponents httpclient - 4.5.13 - - - org.apache.httpcomponents - httpclient-cache - 4.5.13 + ${httpclient.version} org.apache.httpcomponents httpmime - 4.5.13 + ${httpclient.version} org.apache.httpcomponents - fluent-hc - 4.5.13 + httpcore + ${httpcore.version} - org.apache.httpcomponents - httpcore - 4.4.15 + commons-logging + commons-logging + ${commons-logging.version} commons-codec commons-codec - 1.15 - - - com.fasterxml.jackson.core - jackson-core - 2.18.2 + ${commons-codec.version} com.fasterxml.jackson.core jackson-databind - 2.18.2 + ${jackson.databind.version} org.json json - 20250107 + ${json.version} commons-io commons-io - 2.18.0 - - - org.apache.commons - commons-collections4 - 4.4 + ${commons-io.version} com.apicatalog titanium-json-ld - 1.4.0 + ${titanium-json-ld.version} org.glassfish jakarta.json - 2.0.1 + ${jakarta.json.version} + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.10.2 + test diff --git a/src/main/java/org/sead/uploader/AbstractUploader.java b/src/main/java/org/sead/uploader/AbstractUploader.java index bfc71a5..b09cd46 100644 --- a/src/main/java/org/sead/uploader/AbstractUploader.java +++ b/src/main/java/org/sead/uploader/AbstractUploader.java @@ -74,6 +74,30 @@ public abstract class AbstractUploader { protected Set excluded = new HashSet(); protected static List requests = new ArrayList(); + public void clearRequests() { + requests.clear(); + max = Long.MAX_VALUE; + skip = 0l; + globalFileCount = 0l; + totalBytes = 0l; + listonly = false; + verify = false; + importRO = false; + merge = true; + excluded.clear(); + hashIssues.clear(); + roDataIdToNewId.clear(); + roCollIdToNewId.clear(); + roFolderProxy.clear(); + server = null; + bagLocation = null; + clearCache(); + } + + public void clearCache() { + // To be overridden by subclasses to clear internal caches + } + protected static String server = null; PrintWriter pw = null; @@ -117,7 +141,11 @@ public static void printStatus(float s) { public void parseArgs(String[] args) { for (String arg : args) { - // println("Arg is : " + arg); + if (arg.startsWith("-key" + argSeparator)) { + println("Arg is : -key" + argSeparator + "MASKED"); + } else { + println("Arg is : " + arg); + } if (arg.equalsIgnoreCase("-listonly")) { listonly = true; println("List Only Mode"); @@ -206,8 +234,6 @@ public void processRequests() { // the // collection postProcessCollection(); - } else { - postProcessChildren(file); } } else { newUri = null; // listonly - report no changes diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index 334e704..11ae3d5 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -50,6 +50,7 @@ import org.apache.commons.codec.binary.Hex; import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; @@ -108,6 +109,13 @@ public class DVUploader extends AbstractUploader { private int timeout = 1200; private int httpConcurrency = 4; + private static int uploadUrlMaxRetries = 5; + private static int uploadUrlBaseRetryDelayMs = 2000; + private static int uploadUrlMaxRetryDelayMs = 60000; + private static long uploadUrlCooldownUntil = 0; + private static int uploadUrlInterRequestDelayMs = 0; + private static long lastUploadUrlRequestTimestamp = 0; + //private static long mpSizeLimit = 5 * 1024 * 1024; private RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) @@ -154,7 +162,7 @@ public static void main(String args[]) throws Exception { private static void usage() { println("\nUsage:"); - println(" java -jar DVUploader-1.2.0.jar -server= -key= -did= "); + println(" java -jar DVUploader-v1.4.0.jar -server= -key= -did= "); println("\n where:"); println(" = the URL of the server to upload to, e.g. https://datverse.tdl.org"); @@ -186,7 +194,7 @@ public boolean parseCustomArg(String arg) { if (arg.startsWith("-key")) { apiKey = arg.substring(arg.indexOf(argSeparator) + 1); - println("Using apiKey: " + apiKey); + println("Using apiKey: MASKED"); return true; } else if (arg.startsWith("-did")) { datasetPID = arg.substring(arg.indexOf(argSeparator) + 1); @@ -254,7 +262,7 @@ public void processRequests() { String serviceUrl = server + "/api/files/fixityAlgorithm"; HttpGet httpget = new HttpGet(serviceUrl); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); try { switch (response.getStatusLine().getStatusCode()) { case 200: @@ -302,6 +310,107 @@ public HttpClientContext authenticate() { return new HttpClientContext(); } + private static synchronized void updateUploadUrlCooldown(long delayMs) { + uploadUrlCooldownUntil = Math.max(uploadUrlCooldownUntil, System.currentTimeMillis() + delayMs); + } + + private static synchronized void recordUploadUrlRequest() { + lastUploadUrlRequestTimestamp = System.currentTimeMillis(); + } + + private static synchronized void waitForUploadUrlCooldown() { + long now = System.currentTimeMillis(); + long waitTime = Math.max(uploadUrlCooldownUntil - now, (lastUploadUrlRequestTimestamp + uploadUrlInterRequestDelayMs) - now); + if (waitTime > 0) { + try { + // println("Waiting for cooldown: " + waitTime + "ms"); + Thread.sleep(waitTime); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + private static long getRetryAfterDelayMs(CloseableHttpResponse response) { + org.apache.http.Header header = response.getFirstHeader("Retry-After"); + if (header != null) { + try { + // Can be a number of seconds or an HTTP-date + String value = header.getValue(); + if (value.matches("\\d+")) { + return Long.parseLong(value) * 1000; + } + // Handle HTTP-date if necessary, but most APIs use seconds + } catch (Exception e) { + // Ignore + } + } + return 0; + } + + public static CloseableHttpResponse executeWithRetry(org.apache.http.client.methods.HttpUriRequest request, CloseableHttpClient client, HttpClientContext context) throws IOException { + int retryCount = 0; + while (true) { + boolean isDatasetApi = request.getURI().getPath().contains("/api/datasets"); + if (isDatasetApi) { + waitForUploadUrlCooldown(); + recordUploadUrlRequest(); + } + + CloseableHttpResponse response = client.execute(request, context); + int status = response.getStatusLine().getStatusCode(); + + if (isDatasetApi && status == 429 && retryCount < uploadUrlMaxRetries) { + long retryAfterDelayMs = getRetryAfterDelayMs(response); + long recoveryDelayMs = Math.max( + retryAfterDelayMs, + Math.min(uploadUrlBaseRetryDelayMs * (long) Math.pow(2, retryCount), (long) uploadUrlMaxRetryDelayMs) + ); + + synchronized (DVUploader.class) { + uploadUrlInterRequestDelayMs += 50; + } + updateUploadUrlCooldown(recoveryDelayMs); + + EntityUtils.consumeQuietly(response.getEntity()); + response.close(); + + println("Retrying call to " + request.getURI() + " due to 429 in " + recoveryDelayMs + "ms (attempt " + (retryCount + 1) + " of " + uploadUrlMaxRetries + ")"); + try { + Thread.sleep(recoveryDelayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during retry wait", e); + } + retryCount++; + continue; + } + + if (!isDatasetApi && status >= 500 && status <= 599 && retryCount < 3) { + HttpEntity entity = (request instanceof HttpEntityEnclosingRequest) ? ((HttpEntityEnclosingRequest) request).getEntity() : null; + if (entity == null || entity.isRepeatable()) { + long baseDelay = 100; + long delay = retryCount == 0 ? baseDelay : baseDelay * (long) Math.pow(2, retryCount); + + EntityUtils.consumeQuietly(response.getEntity()); + response.close(); + + println("Retrying call to " + request.getURI() + " due to " + status + " in " + delay + "ms (attempt " + (retryCount + 1) + " of 3)"); + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during retry wait", e); + } + retryCount++; + continue; + } + } + + return response; + } + } + public CloseableHttpClient getSharedHttpClient() { if (httpclient == null) { try { @@ -318,8 +427,41 @@ public CloseableHttpClient getSharedHttpClient() { return httpclient; } + @Override + public void clearCache() { + datasetMDRetrieved = false; + existingItems = null; + hashIssues.clear(); + apiKey = null; + datasetPID = null; + alias = null; + oldServer = false; + maxWaitTime = 60; + recurse = false; + directUpload = true; + trustCerts = false; + singleFile = false; + noIngest = false; + fixNames = true; + httpclient = null; + cm = null; + fixityAlgorithm = "MD5"; + + // Reset retry configuration to defaults + // These can't be changed via command line currently but are setable if DVUploader is used as a library (as in tests) + uploadUrlMaxRetries = 5; + uploadUrlBaseRetryDelayMs = 2000; + uploadUrlMaxRetryDelayMs = 60000; + + // Note: We intentionally do NOT reset uploadUrlCooldownUntil, + // uploadUrlInterRequestDelayMs, and lastUploadUrlRequestTimestamp here. + // These values represent the current rate-limiting state of the Dataverse + // server and should persist across cache clears (e.g., between tests) + // to avoid hitting rate limits repeatedly in a short time window. + } + + private boolean datasetMDRetrieved = false; HashMap existingItems = null; - boolean datasetMDRetrieved = false; CloseableHttpClient httpclient = null; @@ -365,7 +507,7 @@ public String itemExists(String path, Resource item) { + "&persistentId=" + datasetPID; HttpGet httpget = new HttpGet(serviceUrl); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); JSONArray datafileList = null; try { switch (response.getStatusLine().getStatusCode()) { @@ -537,7 +679,7 @@ private String createDataset(Resource dir, String path) { httppost.setEntity(se); httppost.addHeader("Content-Type","application/json-ld"); - CloseableHttpResponse response = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httppost, httpclient, getLocalContext()); try { if (response.getStatusLine().getStatusCode() == 201) { HttpEntity resEntity = response.getEntity(); @@ -610,7 +752,7 @@ protected void postProcessChildren(Resource dir) { HttpEntity reqEntity = meb.build(); httppost.setEntity(reqEntity); try { - CloseableHttpResponse postResponse = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse postResponse = executeWithRetry(httppost, httpclient, getLocalContext()); int postStatus = postResponse.getStatusLine().getStatusCode(); String postRes = null; @@ -630,6 +772,15 @@ protected void postProcessChildren(Resource dir) { if (fileResult.has("error Code: ")) { errArray.put(fileResult); errIds.add(fileResult.getString("storageIdentifier")); + } else { + // Successfully added - clear metadata to avoid re-registration + for (Resource r : dir.listResources()) { + if (!r.isDirectory() && r.getMetadata().has("storageIdentifier") && + r.getMetadata().getString("storageIdentifier").equals(fileResult.getString("storageIdentifier"))) { + r.setMetadata(new org.json.JSONObject()); + break; + } + } } } println((jsonData.length() - errIds.size()) + " files successfully added from this folder"); @@ -722,7 +873,7 @@ protected void postProcessCollection() { httppost.setEntity(body); - CloseableHttpResponse response = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httppost, httpclient, getLocalContext()); int status = response.getStatusLine().getStatusCode(); String res = null; @@ -837,7 +988,7 @@ protected String uploadDatafile(Resource file, String path) { HttpEntity reqEntity = meb.build(); httppost.setEntity(reqEntity); - CloseableHttpResponse response = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httppost, httpclient, getLocalContext()); try { int status = response.getStatusLine().getStatusCode(); String res = null; @@ -909,7 +1060,7 @@ private boolean isLocked() { urlString = urlString + "?persistentId=" + datasetPID + "&key=" + apiKey; HttpGet httpget = new HttpGet(urlString); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); try { if (response.getStatusLine().getStatusCode() == 200) { HttpEntity resEntity = response.getEntity(); @@ -968,7 +1119,7 @@ private String multipartDirectFileUpload(Resource file, String path, int retries String urlString = server + "/api/datasets/:persistentId/uploadurls"; urlString = urlString + "?persistentId=" + datasetPID + "&key=" + apiKey + "&size=" + file.length(); HttpGet httpget = new HttpGet(urlString); - CloseableHttpResponse response = httpclient.execute(httpget, getLocalContext()); + CloseableHttpResponse response = executeWithRetry(httpget, httpclient, getLocalContext()); try { int status = response.getStatusLine().getStatusCode(); @@ -1000,14 +1151,14 @@ private String multipartDirectFileUpload(Resource file, String path, int retries MessageDigest messageDigest = MessageDigest.getInstance(fixityAlgorithm); try (InputStream inStream = file.getInputStream(); DigestInputStream digestInputStream = new DigestInputStream(inStream, messageDigest)) { - // This is hte new form for requests - keeping the example but won't update until we can change all - //HttpUriRequest httpput = RequestBuilder.put() + // This is the new form for requests - keeping the example but won't update until we can change all + // HttpUriRequest httpput = RequestBuilder.put() // .setUri(uploadUrl) // .setHeader("x-amz-tagging", "dv-state=temp") // .setEntity(new InputStreamEntity(digestInputStream, file.length())) // .build(); httpput.setEntity(new InputStreamEntity(digestInputStream, file.length())); - CloseableHttpResponse putResponse = httpclient.execute(httpput); + CloseableHttpResponse putResponse = executeWithRetry(httpput, httpclient, getLocalContext()); try { int putStatus = putResponse.getStatusLine().getStatusCode(); String putRes = null; @@ -1144,7 +1295,7 @@ private String multipartDirectFileUpload(Resource file, String path, int retries completeUpload.setEntity(body); completeUpload.setHeader("Content-type", "application/json"); - response = httpclient.execute(completeUpload, getLocalContext()); + response = executeWithRetry(completeUpload, httpclient, getLocalContext()); EntityUtils.consumeQuietly(response.getEntity()); status = response.getStatusLine().getStatusCode(); if (status == 200) { @@ -1180,7 +1331,7 @@ private String multipartDirectFileUpload(Resource file, String path, int retries retries = 0; } else { HttpDelete delete = new HttpDelete(server + abortUrl + "&key=" + apiKey); - response = httpclient.execute(delete, getLocalContext()); + response = executeWithRetry(delete, httpclient, getLocalContext()); EntityUtils.consumeQuietly(response.getEntity()); status = response.getStatusLine().getStatusCode(); if (status != 204) { @@ -1257,7 +1408,7 @@ private String registerFileWithDataverse(Resource file, String path, String stor HttpEntity reqEntity = meb.build(); httppost.setEntity(reqEntity); try { - CloseableHttpResponse postResponse = httpclient.execute(httppost, getLocalContext()); + CloseableHttpResponse postResponse = executeWithRetry(httppost, httpclient, getLocalContext()); int postStatus = postResponse.getStatusLine().getStatusCode(); String postRes = null; diff --git a/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java b/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java index 596e354..63cb862 100644 --- a/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java +++ b/src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java @@ -63,17 +63,18 @@ public HttpPartUploadJob(int partNo, String url, Resource file, long size, Map e * @see java.lang.Runnable#run() */ public void run() { - int retries = 3; + int retryCount = 0; + int maxRetries = 3; //println("Starting upload of part: " + partNo); - while (retries > 0) { - if(retries <3) { - println("Retrying upload of part: " + partNo); + while (retryCount < maxRetries) { + if (retryCount > 0) { + println("Retrying upload of part: " + partNo + " (attempt " + (retryCount + 1) + " of " + maxRetries + ")"); } try (InputStream is = file.getInputStream((partNo - 1) * partSize, size)) { HttpPut httpput = new HttpPut(signedUrl); httpput.setEntity(new InputStreamEntity(is, size)); - CloseableHttpResponse putResponse = httpClient.execute(httpput); + CloseableHttpResponse putResponse = DVUploader.executeWithRetry(httpput, httpClient, localContext); int putStatus = putResponse.getStatusLine().getStatusCode(); String putRes = null; HttpEntity putEntity = putResponse.getEntity(); @@ -83,27 +84,32 @@ public void run() { if (putStatus == 200) { //Part successfully stored - parse the eTag from the response and it it to the Map String eTag = putResponse.getFirstHeader("ETag").getValue(); - eTag= eTag.replace("\"",""); + eTag = eTag.replace("\"", ""); eTags.put(Integer.toString(partNo), eTag); - retries = 0; - //println("Completed upload of part: " + partNo); + return; } else { - if (putStatus >= 500) { - println("Upload of part: " + partNo + " failed with status: " + putStatus + " (skipping)"); + if (putStatus >= 500 && putStatus <= 599) { + long baseDelay = 100; + long delay = retryCount == 0 ? baseDelay : baseDelay * (long) Math.pow(2, retryCount); + println("Upload of part: " + partNo + " failed with status: " + putStatus + ". Retrying in " + delay + "ms"); println("Error response: " + putResponse.getStatusLine() + " : " + putRes); - retries--; + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + retryCount++; } else { println("Upload of part: " + partNo + " failed with status: " + putStatus + " (retrying)"); println("Error response: " + putResponse.getStatusLine() + " : " + putRes); - - retries--; + retryCount++; } } } catch (IOException e) { e.printStackTrace(System.out); println("Error uploading part: " + partNo + " : " + e.getMessage()); - retries--; + retryCount++; } } } diff --git a/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java new file mode 100644 index 0000000..93968b7 --- /dev/null +++ b/src/test/java/org/sead/uploader/dataverse/DVUploaderTest.java @@ -0,0 +1,285 @@ +package org.sead.uploader.dataverse; + +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.*; +import org.sead.uploader.util.UploaderException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Basic tests for DVUploader. + * These tests require a live Dataverse instance and valid credentials. + * Configure via test.properties file or environment variables. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class DVUploaderTest { + + private String server; + private String apiKey; + private String datasetPID; + private long partSize; + private DVUploader uploader; + + @BeforeAll + public void setup() throws IOException { + Properties props = new Properties(); + File propFile = new File("test.properties"); + if (propFile.exists()) { + try (FileInputStream fis = new FileInputStream(propFile)) { + props.load(fis); + } + } + + server = System.getProperty("dataverse.server", props.getProperty("dataverse.server", System.getenv("DATAVERSE_SERVER"))); + apiKey = System.getProperty("dataverse.api_key", props.getProperty("dataverse.api_key", System.getenv("DATAVERSE_API_KEY"))); + datasetPID = System.getProperty("dataverse.dataset_pid", props.getProperty("dataverse.dataset_pid", System.getenv("DATAVERSE_DATASET_PID"))); + + String partSizeStr = System.getProperty("dataverse.part_size", props.getProperty("dataverse.part_size", System.getenv("DATAVERSE_PART_SIZE"))); + if (partSizeStr != null) { + partSize = Long.parseLong(partSizeStr); + } else { + partSize = 5 * 1024 * 1024; // Default 5MB + } + + // Skip tests if configuration is missing + Assumptions.assumeTrue(server != null && apiKey != null && datasetPID != null, + "Test configuration missing. Provide dataverse.server, dataverse.api_key, and dataverse.dataset_pid " + + "via test.properties, system properties, or environment variables."); + + uploader = new DVUploader(); + DVUploader.setUploader(uploader); + + // Initial setup of credentials + uploader.parseArgs(new String[]{"-server=" + server, "-key=" + apiKey, "-did=" + datasetPID}); + } + + @BeforeEach + public void resetUploader() { + uploader.clearRequests(); + // Re-apply common args to ensure they are set + uploader.parseArgs(new String[]{"-server=" + server, "-key=" + apiKey, "-did=" + datasetPID}); + } + + @AfterAll + public void cleanup() throws IOException { + if (uploader == null) return; + + System.out.println("Cleaning up uploaded files..."); + CloseableHttpClient httpClient = uploader.getSharedHttpClient(); + + // Get all files in the dataset to find their IDs + String url = server + "/api/datasets/:persistentId/versions/:latest/files?key=" + apiKey + "&persistentId=" + datasetPID; + HttpGet get = new HttpGet(url); + + try (CloseableHttpResponse response = httpClient.execute(get, uploader.getLocalContext())) { + if (response.getStatusLine().getStatusCode() == 200) { + String res = EntityUtils.toString(response.getEntity()); + JSONArray data = new JSONObject(res).getJSONArray("data"); + for (int i = 0; i < data.length(); i++) { + JSONObject fileEntry = data.getJSONObject(i); + JSONObject dataFile = fileEntry.getJSONObject("dataFile"); + String filename = dataFile.getString("filename"); + + // Cleanup any files starting with our test prefixes + if (filename.startsWith("dvuploader-test") || filename.startsWith("dvuploader-large")) { + long id = dataFile.getLong("id"); + deleteFile(httpClient, id); + } + } + } + } + } + + private void deleteFile(CloseableHttpClient httpClient, long id) throws IOException { + String url = server + "/api/files/" + id + "?key=" + apiKey; + HttpDelete delete = new HttpDelete(url); + try (CloseableHttpResponse response = httpClient.execute(delete, uploader.getLocalContext())) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 204 || statusCode == 200) { + System.out.println("Deleted file ID: " + id + " (Status: " + statusCode + ")"); + } else { + System.err.println("Failed to delete file ID: " + id + " Status: " + statusCode); + } + EntityUtils.consumeQuietly(response.getEntity()); + } + } + + @Test + public void testSimpleUpload() throws IOException, UploaderException { + Path tempFile = Files.createTempFile("dvuploader-test", ".txt"); + String filename = tempFile.getFileName().toString(); + Files.writeString(tempFile, "Hello Dataverse!"); + + try { + uploader.parseArgs(new String[]{tempFile.toAbsolutePath().toString()}); + uploader.processRequests(); + + assertTrue(isFileInDataset(filename), "File " + filename + " should be in dataset after upload"); + } finally { + Files.deleteIfExists(tempFile); + } + } + + @Test + public void testDuplicateUpload() throws IOException, UploaderException { + Path tempFile = Files.createTempFile("dvuploader-test-dup", ".txt"); + String filename = tempFile.getFileName().toString(); + Files.writeString(tempFile, "Duplicate Content"); + + try { + // First upload + uploader.parseArgs(new String[]{tempFile.toAbsolutePath().toString()}); + uploader.processRequests(); + assertTrue(isFileInDataset(filename), "File " + filename + " should be in dataset after first upload"); + + // Clear requests and re-add the same file + uploader.clearRequests(); + uploader.parseArgs(new String[]{tempFile.toAbsolutePath().toString()}); + + // Second upload - should see it exists and not re-upload (this is handled internally by DVUploader) + uploader.processRequests(); + + // Verify it's still there and there's only one (by name) + assertEquals(1, countFileInDataset(filename), "There should be exactly one file named " + filename + " in the dataset"); + } finally { + Files.deleteIfExists(tempFile); + } + } + + @Test + public void testLargeFileUpload() throws IOException, UploaderException { + System.out.println("Using part size: " + partSize); + // Create a file slightly larger than partSize to trigger multipart upload + long fileSize = partSize + (1024 * 1024); // partSize + 1 MB + + Path largeFile = Files.createTempFile("dvuploader-large", ".bin"); + String filename = largeFile.getFileName().toString(); + + System.out.println("Creating " + fileSize + " bytes temp file: " + filename); + try (OutputStream os = Files.newOutputStream(largeFile)) { + byte[] buffer = new byte[1024 * 1024]; // 1MB buffer + long written = 0; + while (written < fileSize) { + int toWrite = (int) Math.min(buffer.length, fileSize - written); + os.write(buffer, 0, toWrite); + written += toWrite; + } + } + + try { + uploader.parseArgs(new String[]{largeFile.toAbsolutePath().toString()}); + uploader.processRequests(); + assertTrue(isFileInDataset(filename), "Large file " + filename + " should be in dataset after upload"); + } finally { + Files.deleteIfExists(largeFile); + } + } + + @Test + public void testDirectoryTreeUploadWithLimit() throws IOException, UploaderException { + Path tempDir = Files.createTempDirectory("dvuploader-test-tree"); + try { + Path file1 = Files.createFile(tempDir.resolve("dvuploader-test-tree1.txt")); + Files.writeString(file1, "File 1 content"); + Path subDir = Files.createDirectory(tempDir.resolve("subdir")); + Path file2 = Files.createFile(subDir.resolve("dvuploader-test-tree2.txt")); + Files.writeString(file2, "File 2 content"); + Path file3 = Files.createFile(subDir.resolve("dvuploader-test-tree3.txt")); + Files.writeString(file3, "File 3 content"); + + String[] filenames = { + file1.getFileName().toString(), + file2.getFileName().toString(), + file3.getFileName().toString() + }; + + // Run 1: limit = 1, recurse + System.out.println("Run 1: limit=1"); + uploader.parseArgs(new String[]{"-limit=1", "-recurse", tempDir.toAbsolutePath().toString()}); + uploader.processRequests(); + + int count = countFilesFromSet(filenames); + assertEquals(1, count, "Should have exactly 1 file uploaded in first run"); + + // Run 2: no limit, recurse + System.out.println("Run 2: full upload"); + uploader.clearRequests(); + uploader.parseArgs(new String[]{"-server=" + server, "-key=" + apiKey, "-did=" + datasetPID}); // Re-add common args + uploader.parseArgs(new String[]{"-recurse", tempDir.toAbsolutePath().toString()}); + uploader.processRequests(); + + count = countFilesFromSet(filenames); + assertEquals(3, count, "Should have all 3 files uploaded after second run"); + + } finally { + deleteDirectory(tempDir); + } + } + + + private int countFilesFromSet(String[] filenames) throws IOException { + int count = 0; + for (String f : filenames) { + if (isFileInDataset(f)) count++; + } + return count; + } + + private void deleteDirectory(Path path) throws IOException { + if (Files.isDirectory(path)) { + try (Stream entries = Files.list(path)) { + entries.forEach(p -> { + try { + deleteDirectory(p); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + } + Files.deleteIfExists(path); + } + + private boolean isFileInDataset(String filename) throws IOException { + return countFileInDataset(filename) > 0; + } + + private int countFileInDataset(String filename) throws IOException { + CloseableHttpClient httpClient = uploader.getSharedHttpClient(); + String url = server + "/api/datasets/:persistentId/versions/:latest/files?key=" + apiKey + "&persistentId=" + datasetPID; + HttpGet get = new HttpGet(url); + int count = 0; + try (CloseableHttpResponse response = httpClient.execute(get, uploader.getLocalContext())) { + if (response.getStatusLine().getStatusCode() == 200) { + String res = EntityUtils.toString(response.getEntity()); + JSONArray data = new JSONObject(res).getJSONArray("data"); + for (int i = 0; i < data.length(); i++) { + JSONObject fileEntry = data.getJSONObject(i); + if (fileEntry.getJSONObject("dataFile").getString("filename").equals(filename)) { + count++; + } + } + } + EntityUtils.consumeQuietly(response.getEntity()); + } + return count; + } +} diff --git a/src/test/java/org/sead/uploader/dataverse/RetryTest.java b/src/test/java/org/sead/uploader/dataverse/RetryTest.java new file mode 100644 index 0000000..cd41251 --- /dev/null +++ b/src/test/java/org/sead/uploader/dataverse/RetryTest.java @@ -0,0 +1,162 @@ +package org.sead.uploader.dataverse; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RetryTest { + + private HttpServer server; + private int port; + private CloseableHttpClient httpClient; + + @BeforeEach + public void setup() throws IOException, NoSuchFieldException, IllegalAccessException { + server = HttpServer.create(new InetSocketAddress(0), 0); + server.setExecutor(null); + server.start(); + port = server.getAddress().getPort(); + httpClient = HttpClients.createDefault(); + + // Speed up tests by reducing retry delays + setStaticField(DVUploader.class, "uploadUrlBaseRetryDelayMs", 10); + setStaticField(DVUploader.class, "uploadUrlMaxRetryDelayMs", 100); + } + + private void setStaticField(Class clazz, String fieldName, Object value) throws NoSuchFieldException, IllegalAccessException { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(null, value); + } + + private Object getStaticField(Class clazz, String fieldName) throws NoSuchFieldException, IllegalAccessException { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(null); + } + + @AfterEach + public void teardown() throws IOException { + if (server != null) { + server.stop(0); + } + if (httpClient != null) { + httpClient.close(); + } + } + + @Test + public void test429Retry() throws IOException { + AtomicInteger callCount = new AtomicInteger(0); + server.createContext("/api/datasets/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + int count = callCount.incrementAndGet(); + if (count == 1) { + exchange.sendResponseHeaders(429, -1); + } else { + byte[] response = "OK".getBytes(); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + } + }); + + HttpGet request = new HttpGet("http://localhost:" + port + "/api/datasets/test"); + try (CloseableHttpResponse response = DVUploader.executeWithRetry(request, httpClient, null)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(2, callCount.get()); + } + } + + @Test + public void test50xRetry() throws IOException { + AtomicInteger callCount = new AtomicInteger(0); + server.createContext("/s3/test", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + int count = callCount.incrementAndGet(); + if (count <= 2) { + exchange.sendResponseHeaders(503, -1); + } else { + byte[] response = "OK".getBytes(); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + } + }); + + HttpGet request = new HttpGet("http://localhost:" + port + "/s3/test"); + try (CloseableHttpResponse response = DVUploader.executeWithRetry(request, httpClient, null)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(3, callCount.get()); + } + } + + @Test + public void testGlobalSlowdown() throws IOException { + // Reset state + // Since fields are private, I can't reset them easily unless I make them package-private or use reflection. + // But I can just check that it increases. + + server.createContext("/api/datasets/slowdown", new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + exchange.sendResponseHeaders(429, -1); + } + }); + + HttpGet request = new HttpGet("http://localhost:" + port + "/api/datasets/slowdown"); + // We expect it to retry 5 times and then return 429 + try (CloseableHttpResponse response = DVUploader.executeWithRetry(request, httpClient, null)) { + assertEquals(429, response.getStatusLine().getStatusCode()); + } + + // Now try another request, it should wait at least 5 * 50ms = 250ms more than before if we hit 429 5 times. + // Actually, uploadUrlInterRequestDelayMs is increased by 50ms on each 429. + } + + @Test + public void testClearCacheResetsRetryConfigButNotTiming() throws IOException, NoSuchFieldException, IllegalAccessException { + // 1. Manually set some non-default values + setStaticField(DVUploader.class, "uploadUrlMaxRetries", 10); + setStaticField(DVUploader.class, "uploadUrlBaseRetryDelayMs", 5000); + setStaticField(DVUploader.class, "uploadUrlMaxRetryDelayMs", 120000); + + setStaticField(DVUploader.class, "uploadUrlCooldownUntil", 123456789L); + setStaticField(DVUploader.class, "uploadUrlInterRequestDelayMs", 500); + setStaticField(DVUploader.class, "lastUploadUrlRequestTimestamp", 987654321L); + + // 2. Call clearCache + new DVUploader().clearCache(); + + // 3. Verify config is reset + assertEquals(5, getStaticField(DVUploader.class, "uploadUrlMaxRetries"), "uploadUrlMaxRetries should be reset to default"); + assertEquals(2000, getStaticField(DVUploader.class, "uploadUrlBaseRetryDelayMs"), "uploadUrlBaseRetryDelayMs should be reset to default"); + assertEquals(60000, getStaticField(DVUploader.class, "uploadUrlMaxRetryDelayMs"), "uploadUrlMaxRetryDelayMs should be reset to default"); + + // 4. Verify timing state is NOT reset + assertEquals(123456789L, getStaticField(DVUploader.class, "uploadUrlCooldownUntil"), "uploadUrlCooldownUntil should NOT be reset"); + assertEquals(500, getStaticField(DVUploader.class, "uploadUrlInterRequestDelayMs"), "uploadUrlInterRequestDelayMs should NOT be reset"); + assertEquals(987654321L, getStaticField(DVUploader.class, "lastUploadUrlRequestTimestamp"), "lastUploadUrlRequestTimestamp should NOT be reset"); + } +} diff --git a/test.properties.example b/test.properties.example new file mode 100644 index 0000000..705a9ea --- /dev/null +++ b/test.properties.example @@ -0,0 +1,7 @@ +# Dataverse Test Configuration +# Copy this file to test.properties and fill in the values +dataverse.server=https://demo.dataverse.org +dataverse.api_key=YOUR_API_KEY +dataverse.dataset_pid=doi:10.5072/FK2/XXXXXX +# Part size for multipart upload in bytes (e.g. 5242880 for 5MB) +dataverse.part_size=5242880