merged refactor branch

build.gradle

@@ -13,7 +13,6 @@ repositories {
     maven {
         name "omtd"
         url "https://repo.openminted.eu/content/repositories/releases/"
-        allowInsecureProtocol = true
     }
     maven {
         name "pentaho-repo"

scripts/startServer.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-set -x
-cd ../
-./gradlew bootrun

src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java

@@ -20,6 +20,8 @@ public final class ImpalaConnector {
 
     public static String impalaDriver;
     public static String impalaConnectionUrl;
+    public static String oldDatabaseName;
+    public static String databaseName;
     public static String poolName;
     public static int hikariMaxConnectionPoolSize;
     public static int hikariMinIdleConnections;
@@ -27,9 +29,6 @@ public final class ImpalaConnector {
     public static int hikariIdleTimeOut;
     public static int hikariMaxLifetime;
 
-    public static String oldDatabaseName = "pdfaggregation_i";
-    public static String databaseName = "pdfAggregationDatabase";
-
     public static final Lock databaseLock = new ReentrantLock(true);  // This lock is locking the threads trying to execute queries in the database.
 
     public static HikariDataSource hikariDataSource;
@@ -57,6 +56,8 @@ public final class ImpalaConnector {
             if ( !"".equals(impalaDriver) ) { // If not "null" or empty.
                 Class.forName(impalaDriver);
                 impalaConnectionUrl = props.getProperty("spring.impala.url");
+                oldDatabaseName = props.getProperty("spring.impala.oldDatabaseName");
+                databaseName = props.getProperty("spring.impala.databaseName");
                 poolName = props.getProperty("spring.datasource.hikari.pool-name");
                 hikariMaxConnectionPoolSize = Integer.parseInt(props.getProperty("spring.datasource.hikari.maximumPoolSize"));
                 hikariMaxLifetime = Integer.parseInt(props.getProperty("spring.datasource.hikari.maxLifetime"));

src/main/java/eu/openaire/urls_controller/util/FileUtils.java

@@ -516,7 +516,7 @@ public class FileUtils {
             Payload payload = urlReport.getPayload();
             if ( payload != null ) {
                 String fileLocation = payload.getLocation();
-                if ( (fileLocation != null) && (! fileLocation.startsWith(S3ObjectStoreMinIO.endpoint)) )
+                if ( (fileLocation != null) && (! fileLocation.startsWith(S3ObjectStoreMinIO.s3Protocol)) )
                     setUnretrievedFullText(payload);
             }
         }

src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java

@@ -17,7 +17,8 @@ public class S3ObjectStoreMinIO {
 
     private static final Logger logger = LoggerFactory.getLogger(S3ObjectStoreMinIO.class);
 
-    public static String endpoint = null;   // This is useful to be "public", to test file-locations.
+    public static String s3Protocol = "s3://";
+    private static String endpoint = null;
     private static String accessKey = null;
     private static String secretKey = null;
     private static String region = null;
@@ -95,12 +96,11 @@ public class S3ObjectStoreMinIO {
             if ( !bucketExists ) {
                 logger.info("Bucket \"" + bucketName + "\" does not exist! Going to create it..");
                 minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
-            }
-            else
+            } else
                 logger.debug("Bucket \"" + bucketName + "\" already exists.");
         } catch (Exception e) {
             String errorMsg = "Could not create the bucket \"" + bucketName + "\"!";
-            logger.error(errorMsg ,e);
+            logger.error(errorMsg, e);
             System.err.println(errorMsg);
             System.exit(56);
         }
@@ -165,7 +165,12 @@ public class S3ObjectStoreMinIO {
             return null;
         }
 
-        String s3Url = endpoint + "/" + bucketName + "/" + fileObjKeyName;  // Be aware: This url works only if the access to the bucket is public.
+        // The urls of OpenAIRE-S3 do not respond PUBLICLY (that's ok). But they also return a 403-Forbidden when requested from a machine with access to them.
+        // An example for such uls is: https://<DOMAIN>//<BUCKET>/doiboost____::3f2fb79f97627fd94c45e694d2a8aa30.pdf
+        // That type of url is usable only in the test S3-Object-Store.
+        // We prefer the following scheme:  s3://<BUCKET>//doiboost____::3f2fb79f97627fd94c45e694d2a8aa30.pdf
+
+        String s3Url = s3Protocol + bucketName + "/" + fileObjKeyName;
         //logger.debug("Uploaded file \"" + fileObjKeyName + "\". The s3Url is: " + s3Url);
         return s3Url;
     }
@@ -207,6 +212,7 @@ public class S3ObjectStoreMinIO {
             }
         }
 
+        logger.info("Bucket " + bucketName + " was " + (shouldDeleteBucket ? "deleted!" : "emptied!"));
         return true;
     }
 
@@ -222,5 +228,4 @@ public class S3ObjectStoreMinIO {
         return true;
     }
 
-
 }

src/main/resources/application.properties

@@ -21,6 +21,9 @@ spring.impala.url = jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/
 # Note: The "UseNativeQuery" does not work with the PreparedStatements! Also, the url does not work without the ending "/"
 # The username and the password do not matter, since this app is always run in an pre-authenticated machine.
 
+spring.impala.oldDatabaseName = pdfaggregation_i
+spring.impala.databaseName = pdfAggregationDatabase
+
 spring.impala.driver-class-name = com.cloudera.impala.jdbc41.Driver
 
 spring.datasource.hikari.pool-name=ControllerPool