- Add progress-report-log for assignments-processing.
- Code polishing.
This commit is contained in:
parent
18cc9e0e68
commit
01e378ea66
|
@ -37,14 +37,11 @@ public class PublicationsRetrieverPlugin {
|
||||||
|
|
||||||
public static String assignmentsBasePath;
|
public static String assignmentsBasePath;
|
||||||
|
|
||||||
private static String workerId;
|
|
||||||
|
|
||||||
private static CookieStore cookieStore = null;
|
private static CookieStore cookieStore = null;
|
||||||
|
|
||||||
|
|
||||||
public PublicationsRetrieverPlugin(@Value("${info.workerId}")String workerId, @Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) {
|
public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) {
|
||||||
// Specify some configurations
|
// Specify some configurations
|
||||||
this.workerId = workerId;
|
|
||||||
LoaderAndChecker.retrieveDocuments = true;
|
LoaderAndChecker.retrieveDocuments = true;
|
||||||
LoaderAndChecker.retrieveDatasets = false;
|
LoaderAndChecker.retrieveDatasets = false;
|
||||||
ConnSupportUtils.setKnownMimeTypes();
|
ConnSupportUtils.setKnownMimeTypes();
|
||||||
|
@ -89,10 +86,17 @@ public class PublicationsRetrieverPlugin {
|
||||||
throw new RuntimeException("Failed to create the full-texts directory for assignments_" + assignmentRequestCounter + ": " + e.getMessage());
|
throw new RuntimeException("Failed to create the full-texts directory for assignments_" + assignmentRequestCounter + ": " + e.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
final int[] urlsCounter = {0};
|
||||||
|
int numOfAssignments = assignments.size();
|
||||||
|
|
||||||
// Start loading and checking urls.
|
// Start loading and checking urls.
|
||||||
for ( Assignment assignment : assignments )
|
for ( Assignment assignment : assignments )
|
||||||
{
|
{
|
||||||
callableTasks.add(() -> {
|
callableTasks.add(() -> {
|
||||||
|
urlsCounter[0]++;
|
||||||
|
if ( (urlsCounter[0] % 250) == 0 ) // Every 500 urls, display a "progress" message.
|
||||||
|
logger.debug("Assignments_" + assignmentRequestCounter + " progress: will process url-" + urlsCounter[0] + " out of " + numOfAssignments);
|
||||||
|
|
||||||
String id = assignment.getId();
|
String id = assignment.getId();
|
||||||
String url = assignment.getOriginalUrl();
|
String url = assignment.getOriginalUrl();
|
||||||
|
|
||||||
|
@ -103,11 +107,10 @@ public class PublicationsRetrieverPlugin {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( (url = LoaderAndChecker.handleUrlChecks(id, url)) == null ) {
|
if ( (url = LoaderAndChecker.handleUrlChecks(id, url)) == null )
|
||||||
return false;
|
return false;
|
||||||
} // The "url" might have changed (inside "handleUrlChecks()").
|
|
||||||
|
|
||||||
String urlToCheck = url;
|
String urlToCheck = url; // The "url" might have changed (inside "handleUrlChecks()").
|
||||||
String sourceUrl = urlToCheck; // Hold it here for the logging-messages.
|
String sourceUrl = urlToCheck; // Hold it here for the logging-messages.
|
||||||
if ( (urlToCheck = LoaderAndChecker.basicURLNormalizer.filter(sourceUrl)) == null ) {
|
if ( (urlToCheck = LoaderAndChecker.basicURLNormalizer.filter(sourceUrl)) == null ) {
|
||||||
logger.warn("Could not normalize url: " + sourceUrl);
|
logger.warn("Could not normalize url: " + sourceUrl);
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
package eu.openaire.urls_worker;
|
package eu.openaire.urls_worker;
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
//import org.junit.jupiter.api.Test;
|
||||||
import org.springframework.boot.test.context.SpringBootTest;
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
|
|
||||||
@SpringBootTest
|
@SpringBootTest
|
||||||
|
|
Loading…
Reference in New Issue