forked from lsmyrnaios/UrlsController
- Optimize the "WorkerReportResult" and the "ShutdownWorker" requests.
- Improve documentation.
This commit is contained in:
parent
0b1ab5b991
commit
e2776c50d0
14
README.md
14
README.md
|
@ -10,8 +10,8 @@ For interacting with the database we use [Impala](https://impala.apache.org/).<b
|
|||
|
||||
**BulkImport API**:
|
||||
- "**bulkImportFullTexts**" endpoint: **http://\<IP\>:\<PORT\>/api/bulkImportFullTexts?provenance=\<provenance\>&bulkImportDir=\<bulkImportDir\>&shouldDeleteFilesOnFinish={true|false}** <br>
|
||||
This endpoint loads the right configuration with the help of the "provenance" parameter, delegates the processing to a background thread and returns a message with useful information, including the "reportFileID", which can be used at any moment to request a report about the progress of the bulk-import procedure.
|
||||
The processing job starts running after 30-60 minutes and processes the full-texts files inside the given directory, in the following way: it generates the OpenAIRE-IDs, uploads the files to the S3 Object Store, generates and stores the "payload" records in the database. If it is specified, it removes the successfully imported full-texts from the directory.
|
||||
This endpoint loads the right configuration with the help of the "provenance" parameter, delegates the processing to a background thread and immediately returns a message with useful information, including the "reportFileID", which can be used at any moment to request a report about the progress of the bulk-import procedure.<br>
|
||||
The processing job starts running after 30-60 minutes and processes the full-texts files inside the given directory, in the following way: it generates the OpenAIRE-IDs, uploads the files to the S3 Object Store, generates and stores the "payload" records in the database. If it is requested by the user, it removes the successfully imported full-texts from the directory.
|
||||
- "**getBulkImportReport**" endpoint: **http://\<IP\>:\<PORT\>/api/getBulkImportReport?id=\<reportFileID\>** <br>
|
||||
This endpoint returns the bulkImport report, which corresponds to the given reportFileID, in JSON format.
|
||||
<br>
|
||||
|
@ -30,14 +30,16 @@ For interacting with the database we use [Impala](https://impala.apache.org/).<b
|
|||
<br>
|
||||
|
||||
**Shutdown Service API**:
|
||||
- "**shutdownService**" endpoint: **http://\<IP\>:\<PORT\>/api/shutdownService** <br>
|
||||
- "**shutdownService**" endpoint: **http://localhost:\<PORT\>/api/shutdownService** <br>
|
||||
This endpoint sends "shutdownWorker" requests to all the Workers which are actively participating in the Service. The Workers will shut down after finishing their work-in-progress and all full-texts have been either transferred to the Controller or deleted, in case an error has appeared.<br>
|
||||
Once the Workers are about to shut down, they send a "shutdownReport" to the Controller. A scheduling task runs in the Controller, every 2 hours, and if the user has specified that the Controller must shut down and all the Workers participating in the Service have shutdown, then it gently shuts down the Controller.
|
||||
- "**cancelShutdownService**" endpoint: **http://\<IP\>:\<PORT\>/api/cancelShutdownService** <br>
|
||||
- "**cancelShutdownService**" endpoint: **http://localhost:\<PORT\>/api/cancelShutdownService** <br>
|
||||
This endpoint specifies that the Controller will not shut down, and sends "cancelShutdownWorker" requests to all the Workers which are actively participating in the Service (have not shut down yet), so that they can continue to request assignments.
|
||||
<br>
|
||||
<br>
|
||||
<br>
|
||||
|
||||
Note: The Shutdown Service API is accessible by the Controller's host machine.
|
||||
<br>
|
||||
<br>
|
||||
|
||||
**To install and run the application**:
|
||||
- Run ```git clone``` and then ```cd UrlsController```.
|
||||
|
|
|
@ -41,11 +41,13 @@ public class ShutdownServiceImpl implements ShutdownService {
|
|||
}
|
||||
|
||||
|
||||
private static final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
public boolean postShutdownOrCancelRequestToWorker(String workerId, String workerIp, boolean shouldCancel)
|
||||
{
|
||||
String url = "http://" + workerIp + ":1881/api/" + (shouldCancel ? "cancelShutdownWorker" : "shutdownWorker");
|
||||
try {
|
||||
ResponseEntity<?> responseEntity = new RestTemplate().postForEntity(url, null, String.class);
|
||||
ResponseEntity<?> responseEntity = restTemplate.postForEntity(url, null, String.class);
|
||||
int responseCode = responseEntity.getStatusCodeValue();
|
||||
if ( responseCode != HttpStatus.OK.value() ) {
|
||||
logger.error("HTTP-Connection problem with the submission of the \"postShutdownOrCancelRequestToWorker\" of worker \"" + workerId + "\"! Error-code was: " + responseCode);
|
||||
|
|
|
@ -447,6 +447,8 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
}
|
||||
|
||||
|
||||
private static final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
private boolean postReportResultToWorker(String workerId, long assignmentRequestCounter, String errorMsg)
|
||||
{
|
||||
// Get the IP of this worker.
|
||||
|
@ -461,7 +463,7 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
logger.trace("Going to \"postReportResultToWorker\": \"" + workerId + "\", for assignments_" + assignmentRequestCounter + ((errorMsg != null) ? "\nError: " + errorMsg : ""));
|
||||
|
||||
try {
|
||||
ResponseEntity<String> responseEntity = new RestTemplate().postForEntity(url, errorMsg, String.class); // We may pass a "null" entity.
|
||||
ResponseEntity<String> responseEntity = restTemplate.postForEntity(url, errorMsg, String.class); // We may pass a "null" entity.
|
||||
int responseCode = responseEntity.getStatusCodeValue();
|
||||
if ( responseCode != HttpStatus.OK.value() ) {
|
||||
logger.error("HTTP-Connection problem with the submission of the \"postReportResultToWorker\" of worker \"" + workerId + "\" and assignments_" + assignmentRequestCounter + "! Error-code was: " + responseCode);
|
||||
|
|
Loading…
Reference in New Issue