lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

- Add progress-report-log for assignments-processing. 

- Code polishing.
This commit is contained in:
Lampros Smyrnaios 2023-10-05 12:02:52 +03:00
parent 18cc9e0e68
commit 01e378ea66
2 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java
View File

@ -37,14 +37,11 @@ public class PublicationsRetrieverPlugin {
public static String assignmentsBasePath;
private static String workerId;
private static CookieStore cookieStore = null;
public PublicationsRetrieverPlugin(@Value("${info.workerId}")String workerId, @Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) {
public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) {
// Specify some configurations
this.workerId = workerId;
LoaderAndChecker.retrieveDocuments = true;
LoaderAndChecker.retrieveDatasets = false;
ConnSupportUtils.setKnownMimeTypes();
@ -89,10 +86,17 @@ public class PublicationsRetrieverPlugin {
throw new RuntimeException("Failed to create the full-texts directory for assignments_" + assignmentRequestCounter + ": " + e.getMessage());
}
final int[] urlsCounter = {0};
int numOfAssignments = assignments.size();
// Start loading and checking urls.
for ( Assignment assignment : assignments )
{
callableTasks.add(() -> {
urlsCounter[0]++;
if ( (urlsCounter[0] % 250) == 0 ) // Every 500 urls, display a "progress" message.
logger.debug("Assignments_" + assignmentRequestCounter + " progress: will process url-" + urlsCounter[0] + " out of " + numOfAssignments);
String id = assignment.getId();
String url = assignment.getOriginalUrl();
@ -103,11 +107,10 @@ public class PublicationsRetrieverPlugin {
return false;
}
if ( (url = LoaderAndChecker.handleUrlChecks(id, url)) == null ) {
if ( (url = LoaderAndChecker.handleUrlChecks(id, url)) == null )
return false;
} // The "url" might have changed (inside "handleUrlChecks()").
String urlToCheck = url;
String urlToCheck = url; // The "url" might have changed (inside "handleUrlChecks()").
String sourceUrl = urlToCheck; // Hold it here for the logging-messages.
if ( (urlToCheck = LoaderAndChecker.basicURLNormalizer.filter(sourceUrl)) == null ) {
logger.warn("Could not normalize url: " + sourceUrl);

2
src/test/java/eu/openaire/urls_worker/UrlsWorkerApplicationTests.java
View File

@ -1,6 +1,6 @@
package eu.openaire.urls_worker;
import org.junit.jupiter.api.Test;
//import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest