- Integrate the latest changes from the "PublicationsRetriever"-plugin.

- Update dependencies.

build.gradle

@@ -1,12 +1,12 @@
 buildscript {
     ext {
-        springBootVersion = "2.5.3"
-        springSecurityVersion = "5.5.1"
+        springBootVersion = "2.5.4"
+        springSecurityVersion = "5.5.2"
     }
 }
 
 plugins {
-    id 'org.springframework.boot' version '2.5.3'
+    id 'org.springframework.boot' version '2.5.4'
     id 'io.spring.dependency-management' version '1.0.11.RELEASE'
     id 'java'
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

installAndRun.sh

@@ -20,7 +20,7 @@ cd ../ && rm -rf PublicationsRetriever
 cd ../
 echo -e "\nAsking for sudo, in order to verify the installation of 'gradle'..\n"
 
-gradleVersion="7.1.1"
+gradleVersion="7.2"
 
 wget https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip
 

src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java

@@ -115,17 +115,9 @@ public class PublicationsRetrieverPlugin {
                 try {    // Check if it's a docUrl, if not, it gets crawled.
                     HttpConnUtils.connectAndCheckMimeType(id, sourceUrl, urlToCheck, urlToCheck, null, true, isPossibleDocOrDatasetUrl);
                 } catch (Exception e) {
-                    String wasUrlValid = "true";
-                    String couldRetry = "false";
-                    if ( e instanceof RuntimeException ) {
-                        String message = e.getMessage();
-                        if ( message != null) {
-                            if ( message.contains("HTTP 404 Client Error") )
-                                wasUrlValid = "false";
-                            else if ( message.contains("Server Error") || message.contains("HTTP 408") )
-                                couldRetry = "true";	// 	We could retry at a later time, as the HTTP-non-404-errors can be temporal.
-                        }
-                    }
+                    List<String> list = LoaderAndChecker.getWasValidAndCouldRetry(e);
+                    String wasUrlValid = list.get(0);
+                    String couldRetry = list.get(1);
                     UrlUtils.logOutputData(id, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", wasUrlValid, "false", "false", couldRetry);
                 }
                 return true;
@@ -219,7 +211,10 @@ public class PublicationsRetrieverPlugin {
         try {
             return HttpConnUtils.connectAndCheckMimeType("null", urlToCheck, urlToCheck, urlToCheck, null, true, false);    // Sent the < null > in quotes to avoid an NPE in the concurrent data-structures.
         } catch (Exception e) {
-            UrlUtils.logOutputData(null, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", "true", "false", "false", "false");
+            List<String> list = LoaderAndChecker.getWasValidAndCouldRetry(e);
+            String wasUrlValid = list.get(0);
+            String couldRetry = list.get(1);
+            UrlUtils.logOutputData(null, urlToCheck, null, "unreachable", "Discarded at loading time, due to connectivity problems.", null, true, "true", wasUrlValid, "false", "false", couldRetry);
             return false;
         }
     }