- Update dependencies.
- Code optimization and polishing.
This commit is contained in:
parent
952bf7c035
commit
088cf73b30
13
build.gradle
13
build.gradle
|
@ -1,13 +1,13 @@
|
||||||
plugins {
|
plugins {
|
||||||
id 'org.springframework.boot' version '2.7.13'
|
id 'org.springframework.boot' version '2.7.14'
|
||||||
id 'io.spring.dependency-management' version '1.1.0'
|
id 'io.spring.dependency-management' version '1.1.2'
|
||||||
id 'java'
|
id 'java'
|
||||||
}
|
}
|
||||||
|
|
||||||
java {
|
java {
|
||||||
group = 'eu.openaire.urls_worker'
|
group = 'eu.openaire.urls_worker'
|
||||||
version = '2.1.0-SNAPSHOT'
|
version = '2.1.0-SNAPSHOT'
|
||||||
sourceCompatibility = '1.8'
|
sourceCompatibility = JavaVersion.VERSION_1_8
|
||||||
}
|
}
|
||||||
|
|
||||||
repositories {
|
repositories {
|
||||||
|
@ -44,8 +44,11 @@ dependencies {
|
||||||
// https://mvnrepository.com/artifact/com.google.code.gson/gson
|
// https://mvnrepository.com/artifact/com.google.code.gson/gson
|
||||||
implementation 'com.google.code.gson:gson:2.10.1'
|
implementation 'com.google.code.gson:gson:2.10.1'
|
||||||
|
|
||||||
implementation 'org.apache.commons:commons-compress:1.23.0'
|
implementation("org.apache.commons:commons-compress:1.23.0") {
|
||||||
implementation 'com.github.luben:zstd-jni:1.5.5-4' // Even though this is part of the above dependency, it is needed separately as well, specifically here, in the Worker.
|
exclude group: 'com.github.luben', module: 'zstd-jni'
|
||||||
|
}
|
||||||
|
implementation 'com.github.luben:zstd-jni:1.5.5-5' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
|
||||||
|
// Also, for compressing, we strangely need it, otherwise it does not work.
|
||||||
|
|
||||||
testImplementation 'org.springframework.security:spring-security-test'
|
testImplementation 'org.springframework.security:spring-security-test'
|
||||||
testImplementation "org.springframework.boot:spring-boot-starter-test"
|
testImplementation "org.springframework.boot:spring-boot-starter-test"
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
distributionBase=GRADLE_USER_HOME
|
distributionBase=GRADLE_USER_HOME
|
||||||
distributionPath=wrapper/dists
|
distributionPath=wrapper/dists
|
||||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
|
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
|
||||||
networkTimeout=10000
|
|
||||||
validateDistributionUrl=true
|
validateDistributionUrl=true
|
||||||
|
networkTimeout=10000
|
||||||
zipStoreBase=GRADLE_USER_HOME
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
zipStorePath=wrapper/dists
|
zipStorePath=wrapper/dists
|
||||||
|
|
|
@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then
|
||||||
echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>"; exit 2
|
echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>"; exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
gradleVersion="8.2"
|
gradleVersion="8.2.1"
|
||||||
|
|
||||||
shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.
|
shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.
|
||||||
|
|
||||||
|
|
|
@ -102,7 +102,7 @@ public class AssignmentsHandler {
|
||||||
|
|
||||||
public AssignmentsRequest requestAssignments()
|
public AssignmentsRequest requestAssignments()
|
||||||
{
|
{
|
||||||
logger.info("Going to request " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl);
|
logger.info("Going to request up to " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl);
|
||||||
AssignmentsRequest assignmentRequest = null;
|
AssignmentsRequest assignmentRequest = null;
|
||||||
try { // Here, the HTTP-request is executed.
|
try { // Here, the HTTP-request is executed.
|
||||||
assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class);
|
assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class);
|
||||||
|
@ -145,7 +145,7 @@ public class AssignmentsHandler {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " tasks.");
|
logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " assignments.");
|
||||||
|
|
||||||
Instant startTime = Instant.now();
|
Instant startTime = Instant.now();
|
||||||
|
|
||||||
|
@ -154,9 +154,8 @@ public class AssignmentsHandler {
|
||||||
|
|
||||||
assignments = getAssignmentsSpacedOutByDomain(assignments, assignmentsSize, false);
|
assignments = getAssignmentsSpacedOutByDomain(assignments, assignmentsSize, false);
|
||||||
|
|
||||||
// Iterate over the tasks and add each task in its own list depending on the DATASOURCE in order to decide which plugin to use later.
|
// Iterate over the assignments and add each assignment in its own list depending on the DATASOURCE in order to decide which plugin to use later.
|
||||||
for ( Assignment assignment : assignments ) {
|
for ( Assignment assignment : assignments ) {
|
||||||
// Add each task in its own HashSet.
|
|
||||||
try {
|
try {
|
||||||
assignmentsForPlugins.put(assignment.getDatasource().getId(), assignment);
|
assignmentsForPlugins.put(assignment.getDatasource().getId(), assignment);
|
||||||
} catch (NullPointerException npe) {
|
} catch (NullPointerException npe) {
|
||||||
|
@ -166,12 +165,12 @@ public class AssignmentsHandler {
|
||||||
|
|
||||||
//countDatasourcesAndRecords(assignmentsSize); // Only for DEBUG! Keep it commented in normal run.
|
//countDatasourcesAndRecords(assignmentsSize); // Only for DEBUG! Keep it commented in normal run.
|
||||||
|
|
||||||
// TODO - Decide which tasks run with what plugin (depending on their datasource).
|
// TODO - Decide which assignments should run with what plugin (depending on their datasource).
|
||||||
// First run -in parallel- the tasks which require some specific plugin.
|
// First run -in parallel- the assignments which require some specific plugin.
|
||||||
// Then, after the above plugins are finished, run the remaining tasks in the generic plugin (which handles parallelism itself).
|
// Then, after the above plugins are finished, run the remaining assignments in the generic plugin (which handles parallelism itself).
|
||||||
// TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe.
|
// TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe.
|
||||||
|
|
||||||
// For now, let's just run all tasks in the generic plugin.
|
// For now, let's just run all assignments in the generic plugin.
|
||||||
try {
|
try {
|
||||||
publicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values());
|
publicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|
|
@ -59,7 +59,7 @@ public class WorkerReport {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getJsonReport() {
|
public String getJsonReport() {
|
||||||
return gson.toJson(this);
|
return gson.toJson(this, WorkerReport.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue