From 088cf73b30dc8dc8e4133ad4e449787355fb4ecb Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Thu, 27 Jul 2023 17:46:17 +0300 Subject: [PATCH] - Update dependencies. - Code optimization and polishing. --- build.gradle | 13 ++++++++----- gradle/wrapper/gradle-wrapper.properties | 4 ++-- installAndRun.sh | 2 +- .../components/AssignmentsHandler.java | 15 +++++++-------- .../payloads/responces/WorkerReport.java | 2 +- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index 6c1361f..fa206a4 100644 --- a/build.gradle +++ b/build.gradle @@ -1,13 +1,13 @@ plugins { - id 'org.springframework.boot' version '2.7.13' - id 'io.spring.dependency-management' version '1.1.0' + id 'org.springframework.boot' version '2.7.14' + id 'io.spring.dependency-management' version '1.1.2' id 'java' } java { group = 'eu.openaire.urls_worker' version = '2.1.0-SNAPSHOT' - sourceCompatibility = '1.8' + sourceCompatibility = JavaVersion.VERSION_1_8 } repositories { @@ -44,8 +44,11 @@ dependencies { // https://mvnrepository.com/artifact/com.google.code.gson/gson implementation 'com.google.code.gson:gson:2.10.1' - implementation 'org.apache.commons:commons-compress:1.23.0' - implementation 'com.github.luben:zstd-jni:1.5.5-4' // Even though this is part of the above dependency, it is needed separately as well, specifically here, in the Worker. + implementation("org.apache.commons:commons-compress:1.23.0") { + exclude group: 'com.github.luben', module: 'zstd-jni' + } + implementation 'com.github.luben:zstd-jni:1.5.5-5' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often. + // Also, for compressing, we strangely need it, otherwise it does not work. testImplementation 'org.springframework.security:spring-security-test' testImplementation "org.springframework.boot:spring-boot-starter-test" diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 62f495d..20f76b5 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip -networkTimeout=10000 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip validateDistributionUrl=true +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/installAndRun.sh b/installAndRun.sh index eb0a1a1..c3a0aea 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh "; exit 2 fi -gradleVersion="8.2" +gradleVersion="8.2.1" shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg. diff --git a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java index d62e392..4398d86 100644 --- a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java @@ -102,7 +102,7 @@ public class AssignmentsHandler { public AssignmentsRequest requestAssignments() { - logger.info("Going to request " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl); + logger.info("Going to request up to " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl); AssignmentsRequest assignmentRequest = null; try { // Here, the HTTP-request is executed. assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class); @@ -145,7 +145,7 @@ public class AssignmentsHandler { return; } - logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " tasks."); + logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " assignments."); Instant startTime = Instant.now(); @@ -154,9 +154,8 @@ public class AssignmentsHandler { assignments = getAssignmentsSpacedOutByDomain(assignments, assignmentsSize, false); - // Iterate over the tasks and add each task in its own list depending on the DATASOURCE in order to decide which plugin to use later. + // Iterate over the assignments and add each assignment in its own list depending on the DATASOURCE in order to decide which plugin to use later. for ( Assignment assignment : assignments ) { - // Add each task in its own HashSet. try { assignmentsForPlugins.put(assignment.getDatasource().getId(), assignment); } catch (NullPointerException npe) { @@ -166,12 +165,12 @@ public class AssignmentsHandler { //countDatasourcesAndRecords(assignmentsSize); // Only for DEBUG! Keep it commented in normal run. - // TODO - Decide which tasks run with what plugin (depending on their datasource). - // First run -in parallel- the tasks which require some specific plugin. - // Then, after the above plugins are finished, run the remaining tasks in the generic plugin (which handles parallelism itself). + // TODO - Decide which assignments should run with what plugin (depending on their datasource). + // First run -in parallel- the assignments which require some specific plugin. + // Then, after the above plugins are finished, run the remaining assignments in the generic plugin (which handles parallelism itself). // TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe. - // For now, let's just run all tasks in the generic plugin. + // For now, let's just run all assignments in the generic plugin. try { publicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values()); } catch (Exception e) { diff --git a/src/main/java/eu/openaire/urls_worker/payloads/responces/WorkerReport.java b/src/main/java/eu/openaire/urls_worker/payloads/responces/WorkerReport.java index f15923c..0cac145 100644 --- a/src/main/java/eu/openaire/urls_worker/payloads/responces/WorkerReport.java +++ b/src/main/java/eu/openaire/urls_worker/payloads/responces/WorkerReport.java @@ -59,7 +59,7 @@ public class WorkerReport { } public String getJsonReport() { - return gson.toJson(this); + return gson.toJson(this, WorkerReport.class); }