From 92b11baf93249a8b5c500ad35779f914277b4fcd Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Fri, 28 Jan 2022 00:59:19 +0200 Subject: [PATCH 1/2] - Update the repository for the Impala JDBC Driver. - Code cleanup. --- build.gradle | 5 ++--- scripts/startServer.sh | 4 ---- .../eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java | 3 +-- 3 files changed, 3 insertions(+), 9 deletions(-) delete mode 100755 scripts/startServer.sh diff --git a/build.gradle b/build.gradle index f9b87fe..ee05543 100644 --- a/build.gradle +++ b/build.gradle @@ -11,9 +11,8 @@ sourceCompatibility = '1.8' repositories { mavenCentral() maven { - name "icm" - url "http://esperos.di.uoa.gr/repo" - allowInsecureProtocol = true + name "omtd" + url "https://repo.openminted.eu/content/repositories/releases/" } maven { name "pentaho-repo" diff --git a/scripts/startServer.sh b/scripts/startServer.sh deleted file mode 100755 index d5ad099..0000000 --- a/scripts/startServer.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -x -cd ../ -./gradlew bootrun diff --git a/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java b/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java index 642e803..b1fbcef 100644 --- a/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java +++ b/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java @@ -100,7 +100,7 @@ public class S3ObjectStoreMinIO { logger.debug("Bucket \"" + bucketName + "\" already exists."); } catch (Exception e) { String errorMsg = "Could not create the bucket \"" + bucketName + "\"!"; - logger.error(errorMsg ,e); + logger.error(errorMsg, e); System.err.println(errorMsg); System.exit(56); } @@ -222,5 +222,4 @@ public class S3ObjectStoreMinIO { return true; } - } From d0ab42e4fae0b54bb4b1a37948c3ad2efdbde004 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Fri, 28 Jan 2022 07:24:42 +0200 Subject: [PATCH 2/2] - Change the scheme of the file-location URI. - Move the old and the current database names in the "application.properties" file. - Improve logging. --- .../configuration/ImpalaConnector.java | 7 ++++--- .../openaire/urls_controller/util/FileUtils.java | 2 +- .../urls_controller/util/S3ObjectStoreMinIO.java | 14 ++++++++++---- src/main/resources/application.properties | 3 +++ 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java index c0085e6..e6249d4 100644 --- a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java +++ b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java @@ -20,6 +20,8 @@ public final class ImpalaConnector { public static String impalaDriver; public static String impalaConnectionUrl; + public static String oldDatabaseName; + public static String databaseName; public static String poolName; public static int hikariMaxConnectionPoolSize; public static int hikariMinIdleConnections; @@ -27,9 +29,6 @@ public final class ImpalaConnector { public static int hikariIdleTimeOut; public static int hikariMaxLifetime; - public static String oldDatabaseName = "pdfaggregation_i"; - public static String databaseName = "pdfAggregationDatabase"; - public static final Lock databaseLock = new ReentrantLock(true); // This lock is locking the threads trying to execute queries in the database. public static HikariDataSource hikariDataSource; @@ -57,6 +56,8 @@ public final class ImpalaConnector { if ( !"".equals(impalaDriver) ) { // If not "null" or empty. Class.forName(impalaDriver); impalaConnectionUrl = props.getProperty("spring.impala.url"); + oldDatabaseName = props.getProperty("spring.impala.oldDatabaseName"); + databaseName = props.getProperty("spring.impala.databaseName"); poolName = props.getProperty("spring.datasource.hikari.pool-name"); hikariMaxConnectionPoolSize = Integer.parseInt(props.getProperty("spring.datasource.hikari.maximumPoolSize")); hikariMaxLifetime = Integer.parseInt(props.getProperty("spring.datasource.hikari.maxLifetime")); diff --git a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java index 01f66d8..0925cae 100644 --- a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java @@ -516,7 +516,7 @@ public class FileUtils { Payload payload = urlReport.getPayload(); if ( payload != null ) { String fileLocation = payload.getLocation(); - if ( (fileLocation != null) && (! fileLocation.startsWith(S3ObjectStoreMinIO.endpoint)) ) + if ( (fileLocation != null) && (! fileLocation.startsWith(S3ObjectStoreMinIO.s3Protocol)) ) setUnretrievedFullText(payload); } } diff --git a/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java b/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java index b1fbcef..86642be 100644 --- a/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java +++ b/src/main/java/eu/openaire/urls_controller/util/S3ObjectStoreMinIO.java @@ -17,7 +17,8 @@ public class S3ObjectStoreMinIO { private static final Logger logger = LoggerFactory.getLogger(S3ObjectStoreMinIO.class); - public static String endpoint = null; // This is useful to be "public", to test file-locations. + public static String s3Protocol = "s3://"; + private static String endpoint = null; private static String accessKey = null; private static String secretKey = null; private static String region = null; @@ -95,8 +96,7 @@ public class S3ObjectStoreMinIO { if ( !bucketExists ) { logger.info("Bucket \"" + bucketName + "\" does not exist! Going to create it.."); minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); - } - else + } else logger.debug("Bucket \"" + bucketName + "\" already exists."); } catch (Exception e) { String errorMsg = "Could not create the bucket \"" + bucketName + "\"!"; @@ -165,7 +165,12 @@ public class S3ObjectStoreMinIO { return null; } - String s3Url = endpoint + "/" + bucketName + "/" + fileObjKeyName; // Be aware: This url works only if the access to the bucket is public. + // The urls of OpenAIRE-S3 do not respond PUBLICLY (that's ok). But they also return a 403-Forbidden when requested from a machine with access to them. + // An example for such uls is: https://///doiboost____::3f2fb79f97627fd94c45e694d2a8aa30.pdf + // That type of url is usable only in the test S3-Object-Store. + // We prefer the following scheme: s3:////doiboost____::3f2fb79f97627fd94c45e694d2a8aa30.pdf + + String s3Url = s3Protocol + bucketName + "/" + fileObjKeyName; //logger.debug("Uploaded file \"" + fileObjKeyName + "\". The s3Url is: " + s3Url); return s3Url; } @@ -207,6 +212,7 @@ public class S3ObjectStoreMinIO { } } + logger.info("Bucket " + bucketName + " was " + (shouldDeleteBucket ? "deleted!" : "emptied!")); return true; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 04f472c..3736129 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -21,6 +21,9 @@ spring.impala.url = jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/ # Note: The "UseNativeQuery" does not work with the PreparedStatements! Also, the url does not work without the ending "/" # The username and the password do not matter, since this app is always run in an pre-authenticated machine. +spring.impala.oldDatabaseName = pdfaggregation_i +spring.impala.databaseName = pdfAggregationDatabase + spring.impala.driver-class-name = com.cloudera.impala.jdbc41.Driver spring.datasource.hikari.pool-name=ControllerPool