Fix adding an invalid error-message in case of an "alreadyDownloaded" full-text being discovered inside the "FileUtils.dataToBeLoggedList".
This commit is contained in:
parent
0032a8018f
commit
8912bb1cf9
|
@ -166,6 +166,7 @@ public class PublicationsRetrieverPlugin {
|
||||||
String initialSourceUrl = comment.substring(indexOfAlreadyDownloadedFromSourceUrl);
|
String initialSourceUrl = comment.substring(indexOfAlreadyDownloadedFromSourceUrl);
|
||||||
//logger.debug("initialId: " + initialId + " | sourceUrl: " + initialSourceUrl); // DEBUG!
|
//logger.debug("initialId: " + initialId + " | sourceUrl: " + initialSourceUrl); // DEBUG!
|
||||||
// Search that ID and sourceUrl inside the list, if that instance is the first-found one, then get the file-data (there might be duplicate ID-sourceUrl instances, but only one of them has the file-data).
|
// Search that ID and sourceUrl inside the list, if that instance is the first-found one, then get the file-data (there might be duplicate ID-sourceUrl instances, but only one of them has the file-data).
|
||||||
|
boolean foundAlreadyDownloadedFullText = false;
|
||||||
for ( DataToBeLogged data_2 : FileUtils.dataToBeLoggedList ) {
|
for ( DataToBeLogged data_2 : FileUtils.dataToBeLoggedList ) {
|
||||||
if ( data_2.getUrlId().equals(initialId) && (data_2.getSourceUrl().equals(initialSourceUrl))
|
if ( data_2.getUrlId().equals(initialId) && (data_2.getSourceUrl().equals(initialSourceUrl))
|
||||||
&& ! data_2.getComment().startsWith(UrlUtils.alreadyDownloadedFromIDMessage) ) {
|
&& ! data_2.getComment().startsWith(UrlUtils.alreadyDownloadedFromIDMessage) ) {
|
||||||
|
@ -173,11 +174,13 @@ public class PublicationsRetrieverPlugin {
|
||||||
size = data_2.getSize();
|
size = data_2.getSize();
|
||||||
hash = data_2.getHash();
|
hash = data_2.getHash();
|
||||||
mimeType = "application/pdf"; // TODO - If support is added for other doc-formats other than "pdf", then make sure the "mime_type" is correctly specified.
|
mimeType = "application/pdf"; // TODO - If support is added for other doc-formats other than "pdf", then make sure the "mime_type" is correctly specified.
|
||||||
|
foundAlreadyDownloadedFullText = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// In case the "alreadyDownloaded" full-text is not found, we have an error.
|
// In case the "alreadyDownloaded" full-text is not found, we have an error.
|
||||||
error = new Error(Error.ErrorType.couldRetry, comment + " | That ID-sourceUrl was not found inside the WorkerReport!"); // We can still try to download it from the found docUrl, in the future.
|
if ( !foundAlreadyDownloadedFullText )
|
||||||
|
error = new Error(Error.ErrorType.couldRetry, comment + " | That ID-sourceUrl was not found inside the WorkerReport!"); // We can still try to download it from the found docUrl, in the future.
|
||||||
}
|
}
|
||||||
else if ( ! comment.equals(HttpConnUtils.docFileNotRetrievedMessage) ) { // If it was downloaded without an error.
|
else if ( ! comment.equals(HttpConnUtils.docFileNotRetrievedMessage) ) { // If it was downloaded without an error.
|
||||||
fileLocation = comment; // This is the full-file-path.
|
fileLocation = comment; // This is the full-file-path.
|
||||||
|
|
|
@ -106,19 +106,18 @@ public class AssignmentsHandler {
|
||||||
//countDatasourcesAndRecords(assignmentsSize); // Only for DEBUG! Keep it commented in normal run.
|
//countDatasourcesAndRecords(assignmentsSize); // Only for DEBUG! Keep it commented in normal run.
|
||||||
|
|
||||||
// TODO - Decide which tasks run with what plugin (depending on their datasource).
|
// TODO - Decide which tasks run with what plugin (depending on their datasource).
|
||||||
|
|
||||||
// First run -in parallel- the tasks which require some specific plugin.
|
// First run -in parallel- the tasks which require some specific plugin.
|
||||||
|
|
||||||
// Then run the remaining tasks in the generic plugin (which handles parallelism itself).
|
// Then run the remaining tasks in the generic plugin (which handles parallelism itself).
|
||||||
|
|
||||||
// For now, let's just run all tasks in the generic plugin.
|
// For now, let's just run all tasks in the generic plugin.
|
||||||
|
|
||||||
try {
|
try {
|
||||||
PublicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values());
|
PublicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("Exception when processing the assignments_" + assignmentRequestCounter, e);
|
logger.error("Exception when processing the assignments_" + assignmentRequestCounter, e);
|
||||||
} // In this case, we will either have an empty WorkerReport or a half-filled one. Either way, we want to report back to the Controller.
|
} // In this case, we will either have an empty WorkerReport or a half-filled one. Either way, we want to report back to the Controller.
|
||||||
|
|
||||||
|
// TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe.
|
||||||
|
|
||||||
if ( askForTest ) {
|
if ( askForTest ) {
|
||||||
logger.debug("UrlReports:"); // DEBUG!
|
logger.debug("UrlReports:"); // DEBUG!
|
||||||
for ( UrlReport urlReport : urlReports )
|
for ( UrlReport urlReport : urlReports )
|
||||||
|
|
Loading…
Reference in New Issue