antonis.lempesis
/
UrlsWorker
forked from lsmyrnaios/UrlsWorker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Integrate some changes from the "PublicationsRetrieverPlugin".

This commit is contained in:
Lampros Smyrnaios 2022-01-14 15:13:00 +02:00
parent 8abb260d60
commit d61ff4b6dd
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
View File

@ -101,7 +101,7 @@ public class PublicationsRetrieverPlugin {
} }
if ( UrlUtils.docOrDatasetUrlsWithIDs.containsKey(url) ) { // If we got into an already-found docUrl, log it and return. if ( UrlUtils.docOrDatasetUrlsWithIDs.containsKey(url) ) { // If we got into an already-found docUrl, log it and return.
ConnSupportUtils.handleReCrossedDocUrl(id, url, url, url, logger, true); ConnSupportUtils.handleReCrossedDocUrl(id, url, url, url, true);
return true; return true;
} }
@ -209,13 +209,14 @@ public class PublicationsRetrieverPlugin {
public static boolean connectWithUrlTest(String urlToCheck) { public static boolean connectWithUrlTest(String urlToCheck) {
String testID = "testID";
try { try {
return HttpConnUtils.connectAndCheckMimeType("null", urlToCheck, urlToCheck, urlToCheck, null, true, false); // Sent the < null > in quotes to avoid an NPE in the concurrent data-structures. return HttpConnUtils.connectAndCheckMimeType(testID, urlToCheck, urlToCheck, urlToCheck, null, true, false); // Sent the < null > in quotes to avoid an NPE in the concurrent data-structures.
} catch (Exception e) { } catch (Exception e) {
List<String> list = LoaderAndChecker.getWasValidAndCouldRetry(e); List<String> list = LoaderAndChecker.getWasValidAndCouldRetry(e);
String wasUrlValid = list.get(0); String wasUrlValid = list.get(0);
String couldRetry = list.get(1); String couldRetry = list.get(1);
UrlUtils.logOutputData(null, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", wasUrlValid, "false", "false", couldRetry, null, null); UrlUtils.logOutputData(testID, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", wasUrlValid, "false", "false", couldRetry, null, null);
return false; return false;
} }
} }