diff --git a/build.gradle b/build.gradle index 1168915..cc9bf21 100644 --- a/build.gradle +++ b/build.gradle @@ -1,11 +1,11 @@ buildscript { ext { - springSecurityVersion = "5.5.0" + springSecurityVersion = "5.5.1" } } plugins { - id 'org.springframework.boot' version '2.5.0' + id 'org.springframework.boot' version '2.5.3' id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'java' } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 0f80bbf..05679dc 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java index 62d1143..dacc379 100644 --- a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java @@ -1,6 +1,6 @@ package eu.openaire.urls_worker.components; -import eu.openaire.urls_worker.plugins.publicationsRetriever.PublicationsRetrieverPlugin; +import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin; import eu.openaire.urls_worker.util.AssignmentHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/eu/openaire/urls_worker/plugins/publicationsRetriever/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java similarity index 96% rename from src/main/java/eu/openaire/urls_worker/plugins/publicationsRetriever/PublicationsRetrieverPlugin.java rename to src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java index a8a5877..82c71d2 100644 --- a/src/main/java/eu/openaire/urls_worker/plugins/publicationsRetriever/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java @@ -1,10 +1,11 @@ -package eu.openaire.urls_worker.plugins.publicationsRetriever; +package eu.openaire.urls_worker.plugins; import com.google.common.hash.Hashing; import com.google.common.io.Files; import edu.uci.ics.crawler4j.url.URLCanonicalizer; import eu.openaire.publications_retriever.PublicationsRetriever; import eu.openaire.publications_retriever.util.file.FileUtils; +import eu.openaire.publications_retriever.util.file.S3ObjectStoreMinIO; import eu.openaire.publications_retriever.util.http.ConnSupportUtils; import eu.openaire.publications_retriever.util.http.HttpConnUtils; import eu.openaire.publications_retriever.util.url.DataToBeLogged; @@ -12,7 +13,6 @@ import eu.openaire.publications_retriever.util.url.LoaderAndChecker; import eu.openaire.publications_retriever.util.url.UrlUtils; import eu.openaire.urls_worker.models.Assignment; import eu.openaire.urls_worker.models.Payload; -import eu.openaire.urls_worker.models.Task; import eu.openaire.urls_worker.models.UrlReport; import eu.openaire.urls_worker.util.AssignmentHandler; import org.slf4j.Logger; @@ -50,8 +50,13 @@ public class PublicationsRetrieverPlugin { LoaderAndChecker.retrieveDocuments = true; LoaderAndChecker.retrieveDatasets = false; FileUtils.shouldDownloadDocFiles = true; + FileUtils.shouldUploadFilesToS3 = true; + FileUtils.docFileNameType = FileUtils.DocFileNameType.idName; PublicationsRetriever.targetUrlType = "docUrl"; + if ( FileUtils.shouldUploadFilesToS3 ) + new S3ObjectStoreMinIO(); // Check here on how to create the credentials-file: https://github.com/LSmyrnaios/PublicationsRetriever/blob/master/README.md + int workerThreadsCount = Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier; logger.info("Use " + workerThreadsCount + " worker-threads."); PublicationsRetriever.executor = Executors.newFixedThreadPool(workerThreadsCount); diff --git a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java b/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java index ca75035..ec3f726 100644 --- a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java +++ b/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java @@ -8,7 +8,7 @@ import eu.openaire.urls_worker.models.Assignment; import eu.openaire.urls_worker.models.UrlReport; import eu.openaire.urls_worker.payloads.requests.AssignmentRequest; import eu.openaire.urls_worker.payloads.responces.WorkerReport; -import eu.openaire.urls_worker.plugins.publicationsRetriever.PublicationsRetrieverPlugin; +import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.web.client.RestTemplateBuilder; @@ -31,7 +31,7 @@ public class AssignmentHandler { public static AssignmentRequest requestAssignments() { RestTemplate restTemplate = new RestTemplateBuilder().build(); - String url = "http://localhost:8080/api/urls/test?workerId=" + WorkerConstants.WORKER_ID + "&assignmentsLimit=" + WorkerConstants.ASSIGNMENTS_LIMIT; + String url = "http://localhost:1880/api/urls/test?workerId=" + WorkerConstants.WORKER_ID + "&workerAssignmentsLimit=" + WorkerConstants.ASSIGNMENTS_LIMIT; String json = null; try { json = restTemplate.getForObject(url, String.class); @@ -113,7 +113,7 @@ public class AssignmentHandler { public static boolean postWorkerReport(Long assignmentRequestCounter) { RestTemplate restTemplate = new RestTemplateBuilder().build(); - String url = "http://localhost:8080/api/urls/addWorkerReport"; + String url = "http://localhost:1880/api/urls/addWorkerReport"; try { ResponseEntity responseEntity = restTemplate.postForEntity(url, new WorkerReport(WorkerConstants.WORKER_ID, assignmentRequestCounter, urlReports), String.class); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 94f58f6..22fdc98 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -10,7 +10,7 @@ #server.error.include-stacktrace=never # HTTP CONFIGURATION -server.port = 8081 +server.port = 1881 # Server api path server.servlet.context-path=/api