lsmyrnaios
/
UrlsController
1
0
Fork 1
Code Issues Pull Requests Releases 13 Wiki Activity

- Add/improve log-messages. 

- Set new version.
- Update dependencies.
- Code polishing.
This commit is contained in:
Lampros Smyrnaios 2024-06-17 12:26:42 +03:00
parent ab18ac5ff8
commit c45a172c21
4 changed files with 24 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@ -6,7 +6,7 @@ plugins {
java {
group = 'eu.openaire.urls_controller'
version = '2.7.4-SNAPSHOT'
version = '2.8.0'
sourceCompatibility = JavaVersion.VERSION_1_8
}
@ -77,7 +77,7 @@ dependencies {
}
// https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro
implementation('org.apache.parquet:parquet-avro:1.14.0')
implementation('org.apache.parquet:parquet-avro:1.14.1')
// https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common
implementation("org.apache.hadoop:hadoop-common:$hadoopVersion") {
@ -116,7 +116,7 @@ dependencies {
implementation 'com.google.code.gson:gson:2.11.0'
// https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.13.0'
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.13.1'
testImplementation 'org.springframework.security:spring-security-test'
testImplementation "org.springframework.boot:spring-boot-starter-test"

2
src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java
View File

@ -156,6 +156,8 @@ public class ScheduledTasks {
if ( ! ShutdownController.shouldShutdownService )
return; // Either the service was never instructed to shut down, or the user canceled the request.
logger.info("The user has requested the shutdown of the Service. Going to check if there is still ongoing work to wait for. If not, then the Service will shutdown immediately.");
// Check whether there are still background tasks to be processed. Either workerReport or Bulk-import requests.
int numOfFutures = UrlsController.futuresOfBackgroundTasks.size();
if ( numOfFutures > 0 ) {

34
src/main/java/eu/openaire/urls_controller/controllers/ShutdownController.java
View File

@ -34,30 +34,29 @@ public class ShutdownController {
{
String initMsg = "Received a \"shutdownService\" request ";
String remoteAddr = GenericUtils.getRequestorAddress(request);
initMsg += "from [" + remoteAddr + "]. ";
initMsg += "from IP=[" + remoteAddr + "]. ";
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
if ( responseEntity != null )
return responseEntity;
String endingMsg;
String additionalMsg;
if ( shouldShutdownService ) {
endingMsg = "The controller has already received a \"shutdownService\" request (which was not canceled afterwards).";
logger.info(initMsg + endingMsg);
additionalMsg = "The controller has already received a \"shutdownService\" request (which was not canceled afterwards).";
logger.info(initMsg + additionalMsg);
} else {
shouldShutdownService = true;
endingMsg = "The service will shutdown, after finishing current work.";
additionalMsg = "The service will shutdown, after finishing current work.";
if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
endingMsg += " None of the workers is active in order to post shutdown requests to.";
logger.info(initMsg + endingMsg);
logger.info(initMsg + additionalMsg + " None of the workers is active in order to post shutdown requests to.");
} else {
logger.info(initMsg + endingMsg); // Show this message before the post-operation, as the following code may take some time to finish.
logger.info(initMsg + additionalMsg); // Show this message before the post-operation, as the following code may take some time to finish.
shutdownService.postShutdownOrCancelRequestsToAllWorkers(false);
}
// That's it for now. The workers may take some hours to finish their work (including delivering the full-text files).
// A scheduler monitors the shutdown of the workers. Once all worker have shutdown, the Controller shuts down as well.
}
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
return ResponseEntity.ok().body(additionalMsg + GenericUtils.endOfLine);
}
@ -66,21 +65,20 @@ public class ShutdownController {
{
String initMsg = "Received a \"cancelShutdownService\" request ";
String remoteAddr = GenericUtils.getRequestorAddress(request);
initMsg += "from [" + remoteAddr + "]. ";
initMsg += "from IP=[" + remoteAddr + "]. ";
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
if ( responseEntity != null )
return responseEntity;
shouldShutdownService = false;
String endingMsg = "Any previous \"shutdownService\"-request is canceled.";
String additionalMsg = "Any previous \"shutdownService\"-request is canceled.";
if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
endingMsg += " None of the workers is active in order to post cancel-shutdown requests to.";
logger.info(initMsg + endingMsg);
logger.info(initMsg + additionalMsg + " None of the workers is active in order to post cancel-shutdown requests to.");
} else {
logger.info(initMsg + endingMsg); // Show this message before the post-operation, as the following code may take some time to finish.
logger.info(initMsg + additionalMsg); // Show this message before the post-operation, as the following code may take some time to finish.
shutdownService.postShutdownOrCancelRequestsToAllWorkers(true); // Cancel the shutdown of the workers, if we are able to catch up with them before they have already shutdown..
}
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
return ResponseEntity.ok().body(additionalMsg + GenericUtils.endOfLine);
}
@ -92,7 +90,7 @@ public class ShutdownController {
{
String initMsg = "Received a \"shutdownAllWorkers\" request ";
String remoteAddr = GenericUtils.getRequestorAddress(request);
initMsg += "from [" + remoteAddr + "]. ";
initMsg += "from IP=[" + remoteAddr + "]. ";
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
if ( responseEntity != null )
return responseEntity;
@ -122,7 +120,7 @@ public class ShutdownController {
{
String initMsg = "Received a \"cancelShutdownAllWorkers\" request ";
String remoteAddr = GenericUtils.getRequestorAddress(request);
initMsg += "from [" + remoteAddr + "]. ";
initMsg += "from IP=[" + remoteAddr + "]. ";
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
if ( responseEntity != null )
return responseEntity;
@ -144,7 +142,7 @@ public class ShutdownController {
public ResponseEntity<?> workerShutdownReport(@RequestParam String workerId, HttpServletRequest request)
{
String remoteAddr = GenericUtils.getRequestorAddress(request);
String initMsg = "Received a \"workerShutdownReport\" from worker: \"" + workerId + "\" [IP: " + remoteAddr + "].";
String initMsg = "Received a \"workerShutdownReport\" from worker: \"" + workerId + "\" with IP=[" + remoteAddr + "].";
WorkerInfo workerInfo = UrlsController.workersInfoMap.get(workerId);
if ( workerInfo == null ) {
String errorMsg = "The worker with id \"" + workerId + "\" has not participated in the PDF-Aggregation-Service!";

8
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -271,7 +271,7 @@ public class FileUtils {
}
// Now we append the file-hash, so it is guaranteed that the filename will be unique.
return datasourceId + "/" + openAireID + "::" + hash + dotFileExtension; // This is the fileName to be used in the objectStore, not of the local file!
return (datasourceId + "/" + openAireID + "::" + hash + dotFileExtension); // This is the fileName to be used in the objectStore, not of the local file!
}
@ -280,10 +280,8 @@ public class FileUtils {
final StringBuilder msgStrB = new StringBuilder(500);
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()), StandardCharsets.UTF_8)) ) {
String inputLine;
while ( (inputLine = br.readLine()) != null )
{
if ( !inputLine.isEmpty() )
msgStrB.append(inputLine);
while ( ((inputLine = br.readLine()) != null) && !inputLine.isEmpty() ) {
msgStrB.append(inputLine);
}
return (msgStrB.length() != 0) ? msgStrB.toString() : null; // Make sure we return a "null" on empty string, to better handle the case in the caller-function.
} catch ( IOException ioe ) {