diff --git a/README.md b/README.md index b59a2ff..b67f67b 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,11 @@ It posts the results to the controller, which in turn, puts them in a database.< To install and run the application: - Run ```git clone``` and then ```cd UrlsWorker```. -- Create the file ```S3_minIO_credentials.txt``` , which contains just one line with the ___S3_url___, ___S3_username___, ___S3_password___, ___S3_server_region___ and the ___S3_bucket___, separated by a _comma_ ```,```. -- [Optional] Create the file ```inputData.txt``` , which contains just one line with the ___workerId___ and the ___controller's base api-url___, seperated by a _comma_ ```,``` . For example: ```worker_1,http://IP:PORT/api/```. -- Execute the ```installAndRun.sh``` script. In case the above file (_inputData.txt_) does not exist, it will request the current ___worker's ID___ and the ___Controller's Url___, and it will create the _inputData.txt_ file.
+- Create the file ```S3_minIO_credentials.txt``` , which contains just one line with the ___S3_url___, ___S3_username___, ___S3_password___, ___S3_server_region___ and the ___S3_bucket___, all separated by a _comma_ ```,```. +- [Optional] Create the file ```inputData.txt``` , which contains just one line with the ___workerId___, the __maxAssignmentsLimitPerBatch__, the __maxAssignmentsBatchesToHandleBeforeRestart__ and the ___controller's base api-url___, all seperated by a _comma_ ```,``` . For example: ```worker_1,http://IP:PORT/api/```. +- Execute the ```installAndRun.sh``` script. In case the above file (_inputData.txt_) does not exist, it will request the current ___worker's ID___, the __maxAssignmentsLimitPerBatch__, the __maxAssignmentsBatchesToHandleBeforeRestart__ and the ___Controller's Url___, and it will create the _inputData.txt_ file.
+Note: If the "maxAssignmentsBatchesToHandleBeforeRestart" is zero or negative, then an infinite number of assignments-batches will be handled. That script, installs the [PublicationsRetriever](https://github.com/LSmyrnaios/PublicationsRetriever), as a library and then compiles and runs the whole Application.
If you want to just run the app, then run the script with the argument "1": ```./installAndRun.sh 1```.

diff --git a/installAndRun.sh b/installAndRun.sh index 97f1a22..1f85aa2 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -20,11 +20,14 @@ if [[ ! -f $inputDataFile ]]; then echo -e "\nGive the max-assignments-limit-per-batch for the Worker to handle: " read -r maxAssignmentsLimitPerBatch + echo -e "\nGive the max-assignments-batches to handle before restart: " + read -r maxAssignmentsBatchesToHandleBeforeRestart + echo -e "\nGive the baseUrl of the controller (e.g.: http://IP:PORT/api/):" read -r controllerBaseUrl touch $inputDataFile - echo "$workerId,$maxAssignmentsLimitPerBatch,$controllerBaseUrl" >> $inputDataFile + echo "$workerId,$maxAssignmentsLimitPerBatch,$maxAssignmentsBatchesToHandleBeforeRestart,$controllerBaseUrl" >> $inputDataFile echo -e "\n\n" fi diff --git a/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java b/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java index e29565a..6c627e5 100644 --- a/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java +++ b/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java @@ -2,6 +2,7 @@ package eu.openaire.urls_worker; import eu.openaire.publications_retriever.PublicationsRetriever; import eu.openaire.publications_retriever.util.file.FileUtils; +import eu.openaire.urls_worker.components.ScheduledTasks; import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin; import eu.openaire.urls_worker.util.AssignmentsHandler; import eu.openaire.urls_worker.util.UriBuilder; @@ -90,6 +91,8 @@ public class UrlsWorkerApplication { } } } + + ScheduledTasks.deleteHandledAssignmentsFullTexts(); } @Bean diff --git a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java index dc515bd..a77ac9a 100644 --- a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java @@ -33,22 +33,15 @@ public class ScheduledTasks { @Scheduled(fixedRate = 900_000) // Every 15 mins: 900_000 public void handleNewAssignments() { if ( AssignmentsHandler.isAvailableForWork ) - { - if ( (UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart == 0) // Infinite batches. - || (AssignmentsHandler.numHandledAssignmentsBatches < UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart) ) - AssignmentsHandler.handleAssignments(); - else { - logger.info("The maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart + ") to be handled was reached! Shut down, in order for the external Linux-service to restart on its own.."); - UrlsWorkerApplication.gentleShutdown(); - } - } else { + AssignmentsHandler.handleAssignments(); + else { //logger.debug("The worker is not available for work at the moment.."); // JUST FOR DEBUG! } } - @Scheduled(fixedRate = 7_200_000) // Every 2 hours. - public void deleteHandledAssignmentsFullTexts() + @Scheduled(fixedRate = 43_200_000) // Every 12 hours. + public static void deleteHandledAssignmentsFullTexts() { Set> entrySet = FullTextsController.assignmentsNumsHandledAndLocallyDeleted.entrySet(); if ( entrySet.isEmpty() ) diff --git a/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java index b828a78..4523f63 100644 --- a/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java @@ -3,6 +3,7 @@ package eu.openaire.urls_worker.util; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import eu.openaire.urls_worker.UrlsWorkerApplication; +import eu.openaire.urls_worker.components.ScheduledTasks; import eu.openaire.urls_worker.models.Assignment; import eu.openaire.urls_worker.models.UrlReport; import eu.openaire.urls_worker.payloads.requests.AssignmentsRequest; @@ -133,6 +134,13 @@ public class AssignmentsHandler { // Note: Cannot call this method here retrospectively, as if it runs 100s of times, the memory-stack may break.. // The scheduler will handle calling it every 15 mins, in case the Worker is available for work.. + + if ( AssignmentsHandler.numHandledAssignmentsBatches == UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart ) + { + logger.info("The maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart + ") to be handled was reached! Shut down, in order for the external Linux-service to restart on its own.."); + ScheduledTasks.deleteHandledAssignmentsFullTexts(); + UrlsWorkerApplication.gentleShutdown(); + } }