- Update dependencies.
- Code optimization and polishing.
This commit is contained in:
parent
952bf7c035
commit
088cf73b30
13
build.gradle
13
build.gradle
|
@ -1,13 +1,13 @@
|
|||
plugins {
|
||||
id 'org.springframework.boot' version '2.7.13'
|
||||
id 'io.spring.dependency-management' version '1.1.0'
|
||||
id 'org.springframework.boot' version '2.7.14'
|
||||
id 'io.spring.dependency-management' version '1.1.2'
|
||||
id 'java'
|
||||
}
|
||||
|
||||
java {
|
||||
group = 'eu.openaire.urls_worker'
|
||||
version = '2.1.0-SNAPSHOT'
|
||||
sourceCompatibility = '1.8'
|
||||
sourceCompatibility = JavaVersion.VERSION_1_8
|
||||
}
|
||||
|
||||
repositories {
|
||||
|
@ -44,8 +44,11 @@ dependencies {
|
|||
// https://mvnrepository.com/artifact/com.google.code.gson/gson
|
||||
implementation 'com.google.code.gson:gson:2.10.1'
|
||||
|
||||
implementation 'org.apache.commons:commons-compress:1.23.0'
|
||||
implementation 'com.github.luben:zstd-jni:1.5.5-4' // Even though this is part of the above dependency, it is needed separately as well, specifically here, in the Worker.
|
||||
implementation("org.apache.commons:commons-compress:1.23.0") {
|
||||
exclude group: 'com.github.luben', module: 'zstd-jni'
|
||||
}
|
||||
implementation 'com.github.luben:zstd-jni:1.5.5-5' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
|
||||
// Also, for compressing, we strangely need it, otherwise it does not work.
|
||||
|
||||
testImplementation 'org.springframework.security:spring-security-test'
|
||||
testImplementation "org.springframework.boot:spring-boot-starter-test"
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
|
||||
networkTimeout=10000
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
|
||||
validateDistributionUrl=true
|
||||
networkTimeout=10000
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
|
|
@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then
|
|||
echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>"; exit 2
|
||||
fi
|
||||
|
||||
gradleVersion="8.2"
|
||||
gradleVersion="8.2.1"
|
||||
|
||||
shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.
|
||||
|
||||
|
|
|
@ -102,7 +102,7 @@ public class AssignmentsHandler {
|
|||
|
||||
public AssignmentsRequest requestAssignments()
|
||||
{
|
||||
logger.info("Going to request " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl);
|
||||
logger.info("Going to request up to " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl);
|
||||
AssignmentsRequest assignmentRequest = null;
|
||||
try { // Here, the HTTP-request is executed.
|
||||
assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class);
|
||||
|
@ -145,7 +145,7 @@ public class AssignmentsHandler {
|
|||
return;
|
||||
}
|
||||
|
||||
logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " tasks.");
|
||||
logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " assignments.");
|
||||
|
||||
Instant startTime = Instant.now();
|
||||
|
||||
|
@ -154,9 +154,8 @@ public class AssignmentsHandler {
|
|||
|
||||
assignments = getAssignmentsSpacedOutByDomain(assignments, assignmentsSize, false);
|
||||
|
||||
// Iterate over the tasks and add each task in its own list depending on the DATASOURCE in order to decide which plugin to use later.
|
||||
// Iterate over the assignments and add each assignment in its own list depending on the DATASOURCE in order to decide which plugin to use later.
|
||||
for ( Assignment assignment : assignments ) {
|
||||
// Add each task in its own HashSet.
|
||||
try {
|
||||
assignmentsForPlugins.put(assignment.getDatasource().getId(), assignment);
|
||||
} catch (NullPointerException npe) {
|
||||
|
@ -166,12 +165,12 @@ public class AssignmentsHandler {
|
|||
|
||||
//countDatasourcesAndRecords(assignmentsSize); // Only for DEBUG! Keep it commented in normal run.
|
||||
|
||||
// TODO - Decide which tasks run with what plugin (depending on their datasource).
|
||||
// First run -in parallel- the tasks which require some specific plugin.
|
||||
// Then, after the above plugins are finished, run the remaining tasks in the generic plugin (which handles parallelism itself).
|
||||
// TODO - Decide which assignments should run with what plugin (depending on their datasource).
|
||||
// First run -in parallel- the assignments which require some specific plugin.
|
||||
// Then, after the above plugins are finished, run the remaining assignments in the generic plugin (which handles parallelism itself).
|
||||
// TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe.
|
||||
|
||||
// For now, let's just run all tasks in the generic plugin.
|
||||
// For now, let's just run all assignments in the generic plugin.
|
||||
try {
|
||||
publicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values());
|
||||
} catch (Exception e) {
|
||||
|
|
|
@ -59,7 +59,7 @@ public class WorkerReport {
|
|||
}
|
||||
|
||||
public String getJsonReport() {
|
||||
return gson.toJson(this);
|
||||
return gson.toJson(this, WorkerReport.class);
|
||||
}
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue