antonis.lempesis
/
UrlsWorker
forked from lsmyrnaios/UrlsWorker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- Improve search-accuracy of "alreadyDownloaded" full-texts. 

- Handle the potential error-case of an "alreadyDownloaded" full-text not being discovered inside the "FileUtils.dataToBeLoggedList".
This commit is contained in:
Lampros Smyrnaios 2022-01-17 10:12:48 +02:00
parent d61ff4b6dd
commit 0032a8018f
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
View File

@ -158,13 +158,17 @@ public class PublicationsRetrieverPlugin {
if ( "true".equals(data.getWasDocumentOrDatasetAccessible()) ) // The reversed order defends against a potential NPE. if ( "true".equals(data.getWasDocumentOrDatasetAccessible()) ) // The reversed order defends against a potential NPE.
{ {
status = UrlReport.StatusType.accessible; status = UrlReport.StatusType.accessible;
if ( comment.startsWith(UrlUtils.alreadyDownloadedByIDMessage, 0) ) { if ( comment.startsWith(UrlUtils.alreadyDownloadedFromIDMessage, 0) ) {
// The file of this docUrl was already downloaded by another docUrl. // The file of this docUrl was already downloaded by another docUrl.
String initialId = comment.substring(UrlUtils.alreadyDownloadedByIDMessage.length()); // The fileName starts right after the "message". int indexOfAlreadyDownloadedFromSourceUrlMessage = comment.indexOf(UrlUtils.alreadyDownloadedFromSourceUrlContinuedMessage);
//logger.debug("initialId: " + initialId); // DEBUG! int indexOfAlreadyDownloadedFromSourceUrl = indexOfAlreadyDownloadedFromSourceUrlMessage + UrlUtils.alreadyDownloadedFromSourceUrlContinuedMessage.length();
// Search that ID inside the list and if that instance gave the docUrl (there might be multiple ID instances) then get the file-location. String initialId = comment.substring(UrlUtils.alreadyDownloadedFromIDMessage.length(), indexOfAlreadyDownloadedFromSourceUrlMessage); // The fileName starts right after the "message".
String initialSourceUrl = comment.substring(indexOfAlreadyDownloadedFromSourceUrl);
//logger.debug("initialId: " + initialId + " | sourceUrl: " + initialSourceUrl); // DEBUG!
// Search that ID and sourceUrl inside the list, if that instance is the first-found one, then get the file-data (there might be duplicate ID-sourceUrl instances, but only one of them has the file-data).
for ( DataToBeLogged data_2 : FileUtils.dataToBeLoggedList ) { for ( DataToBeLogged data_2 : FileUtils.dataToBeLoggedList ) {
if ( data_2.getUrlId().equals(initialId) && ("true".equals(data_2.getWasDocumentOrDatasetAccessible())) ) { if ( data_2.getUrlId().equals(initialId) && (data_2.getSourceUrl().equals(initialSourceUrl))
&& ! data_2.getComment().startsWith(UrlUtils.alreadyDownloadedFromIDMessage) ) {
fileLocation = data_2.getComment(); fileLocation = data_2.getComment();
size = data_2.getSize(); size = data_2.getSize();
hash = data_2.getHash(); hash = data_2.getHash();
@ -172,7 +176,8 @@ public class PublicationsRetrieverPlugin {
break; break;
} }
} }
// TODO - The case where the "twin-ID" is not found, should "never" happen. But should we check? How to handle if that is the case..? // In case the "alreadyDownloaded" full-text is not found, we have an error.
error = new Error(Error.ErrorType.couldRetry, comment + " | That ID-sourceUrl was not found inside the WorkerReport!"); // We can still try to download it from the found docUrl, in the future.
} }
else if ( ! comment.equals(HttpConnUtils.docFileNotRetrievedMessage) ) { // If it was downloaded without an error. else if ( ! comment.equals(HttpConnUtils.docFileNotRetrievedMessage) ) { // If it was downloaded without an error.
fileLocation = comment; // This is the full-file-path. fileLocation = comment; // This is the full-file-path.
@ -181,7 +186,7 @@ public class PublicationsRetrieverPlugin {
error = new Error(Error.ErrorType.couldRetry, comment); // We can still try to download it from the found docUrl, in the future. error = new Error(Error.ErrorType.couldRetry, comment); // We can still try to download it from the found docUrl, in the future.
if ( error == null ) // If the file was retrieved, in any time. if ( error == null ) // If the file was retrieved, in any time.
error = new Error(Error.ErrorType.couldRetry, null); // We do not want to send a "null" object, since it just adds more complicated handling in the controller.. error = new Error(Error.ErrorType.couldRetry, null); // We do not want to send a "null" Error-object, since it just adds more complicated handling in the controller..
} }
else { else {
status = UrlReport.StatusType.non_accessible; status = UrlReport.StatusType.non_accessible;