- Add the ability to upload the files on an S3-ObjectStore.

- Change the server's port and the port of the controller-api.
- Update dependencies.

build.gradle

@@ -1,11 +1,11 @@
 buildscript {
     ext {
-        springSecurityVersion = "5.5.0"
+        springSecurityVersion = "5.5.1"
     }
 }
 
 plugins {
-    id 'org.springframework.boot' version '2.5.0'
+    id 'org.springframework.boot' version '2.5.3'
     id 'io.spring.dependency-management' version '1.0.11.RELEASE'
     id 'java'
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java

@@ -1,6 +1,6 @@
 package eu.openaire.urls_worker.components;
 
-import eu.openaire.urls_worker.plugins.publicationsRetriever.PublicationsRetrieverPlugin;
+import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin;
 import eu.openaire.urls_worker.util.AssignmentHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java

@@ -1,10 +1,11 @@
-package eu.openaire.urls_worker.plugins.publicationsRetriever;
+package eu.openaire.urls_worker.plugins;
 
 import com.google.common.hash.Hashing;
 import com.google.common.io.Files;
 import edu.uci.ics.crawler4j.url.URLCanonicalizer;
 import eu.openaire.publications_retriever.PublicationsRetriever;
 import eu.openaire.publications_retriever.util.file.FileUtils;
+import eu.openaire.publications_retriever.util.file.S3ObjectStoreMinIO;
 import eu.openaire.publications_retriever.util.http.ConnSupportUtils;
 import eu.openaire.publications_retriever.util.http.HttpConnUtils;
 import eu.openaire.publications_retriever.util.url.DataToBeLogged;
@@ -12,7 +13,6 @@ import eu.openaire.publications_retriever.util.url.LoaderAndChecker;
 import eu.openaire.publications_retriever.util.url.UrlUtils;
 import eu.openaire.urls_worker.models.Assignment;
 import eu.openaire.urls_worker.models.Payload;
-import eu.openaire.urls_worker.models.Task;
 import eu.openaire.urls_worker.models.UrlReport;
 import eu.openaire.urls_worker.util.AssignmentHandler;
 import org.slf4j.Logger;
@@ -50,8 +50,13 @@ public class PublicationsRetrieverPlugin {
         LoaderAndChecker.retrieveDocuments = true;
         LoaderAndChecker.retrieveDatasets = false;
         FileUtils.shouldDownloadDocFiles = true;
+        FileUtils.shouldUploadFilesToS3 = true;
+        FileUtils.docFileNameType = FileUtils.DocFileNameType.idName;
         PublicationsRetriever.targetUrlType = "docUrl";
 
+        if ( FileUtils.shouldUploadFilesToS3 )
+            new S3ObjectStoreMinIO();   // Check here on how to create the credentials-file: https://github.com/LSmyrnaios/PublicationsRetriever/blob/master/README.md
+
         int workerThreadsCount = Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier;
         logger.info("Use " + workerThreadsCount + " worker-threads.");
         PublicationsRetriever.executor = Executors.newFixedThreadPool(workerThreadsCount);

src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java

@@ -8,7 +8,7 @@ import eu.openaire.urls_worker.models.Assignment;
 import eu.openaire.urls_worker.models.UrlReport;
 import eu.openaire.urls_worker.payloads.requests.AssignmentRequest;
 import eu.openaire.urls_worker.payloads.responces.WorkerReport;
-import eu.openaire.urls_worker.plugins.publicationsRetriever.PublicationsRetrieverPlugin;
+import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.boot.web.client.RestTemplateBuilder;
@@ -31,7 +31,7 @@ public class AssignmentHandler {
     public static AssignmentRequest requestAssignments()
     {
         RestTemplate restTemplate = new RestTemplateBuilder().build();
-        String url = "http://localhost:8080/api/urls/test?workerId=" + WorkerConstants.WORKER_ID + "&assignmentsLimit=" + WorkerConstants.ASSIGNMENTS_LIMIT;
+        String url = "http://localhost:1880/api/urls/test?workerId=" + WorkerConstants.WORKER_ID + "&workerAssignmentsLimit=" + WorkerConstants.ASSIGNMENTS_LIMIT;
         String json = null;
         try {
             json = restTemplate.getForObject(url, String.class);
@@ -113,7 +113,7 @@ public class AssignmentHandler {
     public static boolean postWorkerReport(Long assignmentRequestCounter)
     {
         RestTemplate restTemplate = new RestTemplateBuilder().build();
-        String url = "http://localhost:8080/api/urls/addWorkerReport";
+        String url = "http://localhost:1880/api/urls/addWorkerReport";
 
         try {
             ResponseEntity<String> responseEntity = restTemplate.postForEntity(url, new WorkerReport(WorkerConstants.WORKER_ID, assignmentRequestCounter, urlReports), String.class);

src/main/resources/application.properties

@@ -10,7 +10,7 @@
 #server.error.include-stacktrace=never
 
 # HTTP CONFIGURATION
-server.port = 8081
+server.port = 1881
 
 # Server api path
 server.servlet.context-path=/api