lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 16 Wiki Activity

- Integrate the latest changes from the "PublicationsRetriever"-plugin. 

- Update dependencies.
This commit is contained in:
Lampros Smyrnaios 2021-09-01 19:42:32 +03:00
parent 5bbf422d3b
commit b6d66653f7
4 changed files with 12 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@ -1,12 +1,12 @@
buildscript { buildscript {
ext { ext {
springBootVersion = "2.5.3" springBootVersion = "2.5.4"
springSecurityVersion = "5.5.1" springSecurityVersion = "5.5.2"
} }
} }
plugins { plugins {
id 'org.springframework.boot' version '2.5.3' id 'org.springframework.boot' version '2.5.4'
id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java' id 'java'
} }

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

2
installAndRun.sh
View File

@ -20,7 +20,7 @@ cd ../ && rm -rf PublicationsRetriever
cd ../ cd ../
echo -e "\nAsking for sudo, in order to verify the installation of 'gradle'..\n" echo -e "\nAsking for sudo, in order to verify the installation of 'gradle'..\n"
gradleVersion="7.1.1" gradleVersion="7.2"
wget https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip wget https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip

19
src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
View File

@ -115,17 +115,9 @@ public class PublicationsRetrieverPlugin {
try { // Check if it's a docUrl, if not, it gets crawled. try { // Check if it's a docUrl, if not, it gets crawled.
HttpConnUtils.connectAndCheckMimeType(id, sourceUrl, urlToCheck, urlToCheck, null, true, isPossibleDocOrDatasetUrl); HttpConnUtils.connectAndCheckMimeType(id, sourceUrl, urlToCheck, urlToCheck, null, true, isPossibleDocOrDatasetUrl);
} catch (Exception e) { } catch (Exception e) {
String wasUrlValid = "true"; List<String> list = LoaderAndChecker.getWasValidAndCouldRetry(e);
String couldRetry = "false"; String wasUrlValid = list.get(0);
if ( e instanceof RuntimeException ) { String couldRetry = list.get(1);
String message = e.getMessage();
if ( message != null) {
if ( message.contains("HTTP 404 Client Error") )
wasUrlValid = "false";
else if ( message.contains("Server Error") || message.contains("HTTP 408") )
couldRetry = "true"; // We could retry at a later time, as the HTTP-non-404-errors can be temporal.
}
}
UrlUtils.logOutputData(id, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", wasUrlValid, "false", "false", couldRetry); UrlUtils.logOutputData(id, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", wasUrlValid, "false", "false", couldRetry);
} }
return true; return true;
@ -219,7 +211,10 @@ public class PublicationsRetrieverPlugin {
try { try {
return HttpConnUtils.connectAndCheckMimeType("null", urlToCheck, urlToCheck, urlToCheck, null, true, false); // Sent the < null > in quotes to avoid an NPE in the concurrent data-structures. return HttpConnUtils.connectAndCheckMimeType("null", urlToCheck, urlToCheck, urlToCheck, null, true, false); // Sent the < null > in quotes to avoid an NPE in the concurrent data-structures.
} catch (Exception e) { } catch (Exception e) {
UrlUtils.logOutputData(null, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", "true", "false", "false", "false"); List<String> list = LoaderAndChecker.getWasValidAndCouldRetry(e);
String wasUrlValid = list.get(0);
String couldRetry = list.get(1);
UrlUtils.logOutputData(null, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", wasUrlValid, "false", "false", couldRetry);
return false; return false;
} }
} }