forked from lsmyrnaios/UrlsWorker
- Make sure the handled assignments - full-texts are deleted before the application exits.
- When the user sets the "maxAssignmentsBatchesToHandleBeforeRestart" above zero, shutdown immediately after the last assignments-batch. Do not wait for the next scheduled check. - Allow the user to set the "maxAssignmentsBatchesToHandleBeforeRestart" in the "installAndRun.sh" script. - Increase the "fixedRate" for the "ScheduledTasks.deleteHandledAssignmentsFullTexts()" method to 12 hours. - Update README.md
This commit is contained in:
parent
1ddfd34236
commit
92d011e8a0
|
@ -7,10 +7,11 @@ It posts the results to the controller, which in turn, puts them in a database.<
|
|||
|
||||
To install and run the application:
|
||||
- Run ```git clone``` and then ```cd UrlsWorker```.
|
||||
- Create the file ```S3_minIO_credentials.txt``` , which contains just one line with the ___S3_url___, ___S3_username___, ___S3_password___, ___S3_server_region___ and the ___S3_bucket___, separated by a _comma_ ```,```.
|
||||
- [Optional] Create the file ```inputData.txt``` , which contains just one line with the ___workerId___ and the ___controller's base api-url___, seperated by a _comma_ ```,``` . For example: ```worker_1,http://IP:PORT/api/```.
|
||||
- Execute the ```installAndRun.sh``` script. In case the above file (_inputData.txt_) does not exist, it will request the current ___worker's ID___ and the ___Controller's Url___, and it will create the _inputData.txt_ file.<br>
|
||||
- Create the file ```S3_minIO_credentials.txt``` , which contains just one line with the ___S3_url___, ___S3_username___, ___S3_password___, ___S3_server_region___ and the ___S3_bucket___, all separated by a _comma_ ```,```.
|
||||
- [Optional] Create the file ```inputData.txt``` , which contains just one line with the ___workerId___, the __maxAssignmentsLimitPerBatch__, the __maxAssignmentsBatchesToHandleBeforeRestart__ and the ___controller's base api-url___, all seperated by a _comma_ ```,``` . For example: ```worker_1,http://IP:PORT/api/```.
|
||||
- Execute the ```installAndRun.sh``` script. In case the above file (_inputData.txt_) does not exist, it will request the current ___worker's ID___, the __maxAssignmentsLimitPerBatch__, the __maxAssignmentsBatchesToHandleBeforeRestart__ and the ___Controller's Url___, and it will create the _inputData.txt_ file.<br>
|
||||
|
||||
Note: If the "maxAssignmentsBatchesToHandleBeforeRestart" is zero or negative, then an infinite number of assignments-batches will be handled.
|
||||
That script, installs the [PublicationsRetriever](https://github.com/LSmyrnaios/PublicationsRetriever), as a library and then compiles and runs the whole Application.<br>
|
||||
If you want to just run the app, then run the script with the argument "1": ```./installAndRun.sh 1```.<br>
|
||||
<br>
|
||||
|
|
|
@ -20,11 +20,14 @@ if [[ ! -f $inputDataFile ]]; then
|
|||
echo -e "\nGive the max-assignments-limit-per-batch for the Worker to handle: "
|
||||
read -r maxAssignmentsLimitPerBatch
|
||||
|
||||
echo -e "\nGive the max-assignments-batches to handle before restart: "
|
||||
read -r maxAssignmentsBatchesToHandleBeforeRestart
|
||||
|
||||
echo -e "\nGive the baseUrl of the controller (e.g.: http://IP:PORT/api/):"
|
||||
read -r controllerBaseUrl
|
||||
|
||||
touch $inputDataFile
|
||||
echo "$workerId,$maxAssignmentsLimitPerBatch,$controllerBaseUrl" >> $inputDataFile
|
||||
echo "$workerId,$maxAssignmentsLimitPerBatch,$maxAssignmentsBatchesToHandleBeforeRestart,$controllerBaseUrl" >> $inputDataFile
|
||||
echo -e "\n\n"
|
||||
fi
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@ package eu.openaire.urls_worker;
|
|||
|
||||
import eu.openaire.publications_retriever.PublicationsRetriever;
|
||||
import eu.openaire.publications_retriever.util.file.FileUtils;
|
||||
import eu.openaire.urls_worker.components.ScheduledTasks;
|
||||
import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin;
|
||||
import eu.openaire.urls_worker.util.AssignmentsHandler;
|
||||
import eu.openaire.urls_worker.util.UriBuilder;
|
||||
|
@ -90,6 +91,8 @@ public class UrlsWorkerApplication {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
ScheduledTasks.deleteHandledAssignmentsFullTexts();
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
|
|
@ -33,22 +33,15 @@ public class ScheduledTasks {
|
|||
@Scheduled(fixedRate = 900_000) // Every 15 mins: 900_000
|
||||
public void handleNewAssignments() {
|
||||
if ( AssignmentsHandler.isAvailableForWork )
|
||||
{
|
||||
if ( (UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart == 0) // Infinite batches.
|
||||
|| (AssignmentsHandler.numHandledAssignmentsBatches < UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart) )
|
||||
AssignmentsHandler.handleAssignments();
|
||||
else {
|
||||
logger.info("The maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart + ") to be handled was reached! Shut down, in order for the external Linux-service to restart on its own..");
|
||||
UrlsWorkerApplication.gentleShutdown();
|
||||
}
|
||||
} else {
|
||||
//logger.debug("The worker is not available for work at the moment.."); // JUST FOR DEBUG!
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Scheduled(fixedRate = 7_200_000) // Every 2 hours.
|
||||
public void deleteHandledAssignmentsFullTexts()
|
||||
@Scheduled(fixedRate = 43_200_000) // Every 12 hours.
|
||||
public static void deleteHandledAssignmentsFullTexts()
|
||||
{
|
||||
Set<Map.Entry<Long, Boolean>> entrySet = FullTextsController.assignmentsNumsHandledAndLocallyDeleted.entrySet();
|
||||
if ( entrySet.isEmpty() )
|
||||
|
|
|
@ -3,6 +3,7 @@ package eu.openaire.urls_worker.util;
|
|||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
import eu.openaire.urls_worker.UrlsWorkerApplication;
|
||||
import eu.openaire.urls_worker.components.ScheduledTasks;
|
||||
import eu.openaire.urls_worker.models.Assignment;
|
||||
import eu.openaire.urls_worker.models.UrlReport;
|
||||
import eu.openaire.urls_worker.payloads.requests.AssignmentsRequest;
|
||||
|
@ -133,6 +134,13 @@ public class AssignmentsHandler {
|
|||
|
||||
// Note: Cannot call this method here retrospectively, as if it runs 100s of times, the memory-stack may break..
|
||||
// The scheduler will handle calling it every 15 mins, in case the Worker is available for work..
|
||||
|
||||
if ( AssignmentsHandler.numHandledAssignmentsBatches == UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart )
|
||||
{
|
||||
logger.info("The maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart + ") to be handled was reached! Shut down, in order for the external Linux-service to restart on its own..");
|
||||
ScheduledTasks.deleteHandledAssignmentsFullTexts();
|
||||
UrlsWorkerApplication.gentleShutdown();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue