Skip to content
Open

V1.4 #39

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
/target/
/target/
test.properties
DVUploaderLog_*.log
30 changes: 27 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,30 @@ For Clowder/SEAD-specific information, see https://opensource.ncsa.illinois.edu/

Build:

mvn clean compile assembly:single

Usage: See wiki: https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse
Run the following command to build the project and create the executable JAR:

```bash
mvn clean package -DskipTests
```

This will produce:
- `target/DVUploader-1.4.0.jar`: Standard library JAR.
- `target/DVUploader-v1.4.0.jar`: Executable "fat" JAR containing all dependencies.

Testing:

Basic functionality tests for Dataverse can be run using Maven. These tests require a live Dataverse instance and valid credentials.

1. Copy `test.properties.example` to `test.properties`.
2. Edit `test.properties` and provide your Dataverse server URL, API key, a test Dataset PID (DOI), and the part size for multipart uploads.
3. Run the tests:
```bash
mvn test
```

Alternatively, you can provide configuration via system properties or environment variables:
```bash
mvn test -Ddataverse.server=... -Ddataverse.api_key=... -Ddataverse.dataset_pid=...
```

Usage: See wiki: https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse
102 changes: 69 additions & 33 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,64 @@
<modelVersion>4.0.0</modelVersion>
<groupId>DVUploader</groupId>
<artifactId>DVUploader</artifactId>
<version>1.3.0-beta</version>
<version>1.4.0</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<httpclient.version>4.5.14</httpclient.version>
<httpcore.version>4.4.16</httpcore.version>
<jackson.databind.version>2.22.1</jackson.databind.version>
<jackson.annotations.version>2.22</jackson.annotations.version>
<commons-codec.version>1.22.0</commons-codec.version>
<commons-io.version>2.22.0</commons-io.version>
<json.version>20260522</json.version>
<titanium-json-ld.version>1.4.0</titanium-json-ld.version>
<jakarta.json.version>2.0.1</jakarta.json.version>
<commons-logging.version>1.3.1</commons-logging.version>
</properties>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<version>3.13.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.5</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.4.1</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>org.sead.uploader.dataverse.DVUploader</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<version>3.7.1</version>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<archive>
<manifest>
Expand All @@ -42,67 +81,64 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient-cache</artifactId>
<version>4.5.13</version>
<version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.5.13</version>
<version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>fluent-hc</artifactId>
<version>4.5.13</version>
<artifactId>httpcore</artifactId>
<version>${httpcore.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.15</version>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${commons-logging.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.15</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.18.2</version>
<version>${commons-codec.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.18.2</version>
<version>${jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20250107</version>
<version>${json.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.18.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
<version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>com.apicatalog</groupId>
<artifactId>titanium-json-ld</artifactId>
<version>1.4.0</version>
<version>${titanium-json-ld.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>jakarta.json</artifactId>
<version>2.0.1</version>
<version>${jakarta.json.version}</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.10.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.10.2</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
32 changes: 29 additions & 3 deletions src/main/java/org/sead/uploader/AbstractUploader.java
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,30 @@ public abstract class AbstractUploader {
protected Set<String> excluded = new HashSet<String>();
protected static List<String> requests = new ArrayList<String>();

public void clearRequests() {
requests.clear();
max = Long.MAX_VALUE;
skip = 0l;
globalFileCount = 0l;
totalBytes = 0l;
listonly = false;
verify = false;
importRO = false;
merge = true;
excluded.clear();
hashIssues.clear();
roDataIdToNewId.clear();
roCollIdToNewId.clear();
roFolderProxy.clear();
server = null;
bagLocation = null;
clearCache();
}

public void clearCache() {
// To be overridden by subclasses to clear internal caches
}

protected static String server = null;

PrintWriter pw = null;
Expand Down Expand Up @@ -117,7 +141,11 @@ public static void printStatus(float s) {
public void parseArgs(String[] args) {

for (String arg : args) {
// println("Arg is : " + arg);
if (arg.startsWith("-key" + argSeparator)) {
println("Arg is : -key" + argSeparator + "MASKED");
} else {
println("Arg is : " + arg);
}
if (arg.equalsIgnoreCase("-listonly")) {
listonly = true;
println("List Only Mode");
Expand Down Expand Up @@ -206,8 +234,6 @@ public void processRequests() {
// the
// collection
postProcessCollection();
} else {
postProcessChildren(file);
}
} else {
newUri = null; // listonly - report no changes
Expand Down
Loading