merged refactor branch

This commit is contained in:
Antonis Lempesis 2022-01-30 23:10:06 +02:00
commit e47fd8d97b
6 changed files with 19 additions and 15 deletions

View File

@ -13,7 +13,6 @@ repositories {
maven { maven {
name "omtd" name "omtd"
url "https://repo.openminted.eu/content/repositories/releases/" url "https://repo.openminted.eu/content/repositories/releases/"
allowInsecureProtocol = true
} }
maven { maven {
name "pentaho-repo" name "pentaho-repo"

View File

@ -1,4 +0,0 @@
#!/usr/bin/env bash
set -x
cd ../
./gradlew bootrun

View File

@ -20,6 +20,8 @@ public final class ImpalaConnector {
public static String impalaDriver; public static String impalaDriver;
public static String impalaConnectionUrl; public static String impalaConnectionUrl;
public static String oldDatabaseName;
public static String databaseName;
public static String poolName; public static String poolName;
public static int hikariMaxConnectionPoolSize; public static int hikariMaxConnectionPoolSize;
public static int hikariMinIdleConnections; public static int hikariMinIdleConnections;
@ -27,9 +29,6 @@ public final class ImpalaConnector {
public static int hikariIdleTimeOut; public static int hikariIdleTimeOut;
public static int hikariMaxLifetime; public static int hikariMaxLifetime;
public static String oldDatabaseName = "pdfaggregation_i";
public static String databaseName = "pdfAggregationDatabase";
public static final Lock databaseLock = new ReentrantLock(true); // This lock is locking the threads trying to execute queries in the database. public static final Lock databaseLock = new ReentrantLock(true); // This lock is locking the threads trying to execute queries in the database.
public static HikariDataSource hikariDataSource; public static HikariDataSource hikariDataSource;
@ -57,6 +56,8 @@ public final class ImpalaConnector {
if ( !"".equals(impalaDriver) ) { // If not "null" or empty. if ( !"".equals(impalaDriver) ) { // If not "null" or empty.
Class.forName(impalaDriver); Class.forName(impalaDriver);
impalaConnectionUrl = props.getProperty("spring.impala.url"); impalaConnectionUrl = props.getProperty("spring.impala.url");
oldDatabaseName = props.getProperty("spring.impala.oldDatabaseName");
databaseName = props.getProperty("spring.impala.databaseName");
poolName = props.getProperty("spring.datasource.hikari.pool-name"); poolName = props.getProperty("spring.datasource.hikari.pool-name");
hikariMaxConnectionPoolSize = Integer.parseInt(props.getProperty("spring.datasource.hikari.maximumPoolSize")); hikariMaxConnectionPoolSize = Integer.parseInt(props.getProperty("spring.datasource.hikari.maximumPoolSize"));
hikariMaxLifetime = Integer.parseInt(props.getProperty("spring.datasource.hikari.maxLifetime")); hikariMaxLifetime = Integer.parseInt(props.getProperty("spring.datasource.hikari.maxLifetime"));

View File

@ -516,7 +516,7 @@ public class FileUtils {
Payload payload = urlReport.getPayload(); Payload payload = urlReport.getPayload();
if ( payload != null ) { if ( payload != null ) {
String fileLocation = payload.getLocation(); String fileLocation = payload.getLocation();
if ( (fileLocation != null) && (! fileLocation.startsWith(S3ObjectStoreMinIO.endpoint)) ) if ( (fileLocation != null) && (! fileLocation.startsWith(S3ObjectStoreMinIO.s3Protocol)) )
setUnretrievedFullText(payload); setUnretrievedFullText(payload);
} }
} }

View File

@ -17,7 +17,8 @@ public class S3ObjectStoreMinIO {
private static final Logger logger = LoggerFactory.getLogger(S3ObjectStoreMinIO.class); private static final Logger logger = LoggerFactory.getLogger(S3ObjectStoreMinIO.class);
public static String endpoint = null; // This is useful to be "public", to test file-locations. public static String s3Protocol = "s3://";
private static String endpoint = null;
private static String accessKey = null; private static String accessKey = null;
private static String secretKey = null; private static String secretKey = null;
private static String region = null; private static String region = null;
@ -95,12 +96,11 @@ public class S3ObjectStoreMinIO {
if ( !bucketExists ) { if ( !bucketExists ) {
logger.info("Bucket \"" + bucketName + "\" does not exist! Going to create it.."); logger.info("Bucket \"" + bucketName + "\" does not exist! Going to create it..");
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
} } else
else
logger.debug("Bucket \"" + bucketName + "\" already exists."); logger.debug("Bucket \"" + bucketName + "\" already exists.");
} catch (Exception e) { } catch (Exception e) {
String errorMsg = "Could not create the bucket \"" + bucketName + "\"!"; String errorMsg = "Could not create the bucket \"" + bucketName + "\"!";
logger.error(errorMsg ,e); logger.error(errorMsg, e);
System.err.println(errorMsg); System.err.println(errorMsg);
System.exit(56); System.exit(56);
} }
@ -165,7 +165,12 @@ public class S3ObjectStoreMinIO {
return null; return null;
} }
String s3Url = endpoint + "/" + bucketName + "/" + fileObjKeyName; // Be aware: This url works only if the access to the bucket is public. // The urls of OpenAIRE-S3 do not respond PUBLICLY (that's ok). But they also return a 403-Forbidden when requested from a machine with access to them.
// An example for such uls is: https://<DOMAIN>//<BUCKET>/doiboost____::3f2fb79f97627fd94c45e694d2a8aa30.pdf
// That type of url is usable only in the test S3-Object-Store.
// We prefer the following scheme: s3://<BUCKET>//doiboost____::3f2fb79f97627fd94c45e694d2a8aa30.pdf
String s3Url = s3Protocol + bucketName + "/" + fileObjKeyName;
//logger.debug("Uploaded file \"" + fileObjKeyName + "\". The s3Url is: " + s3Url); //logger.debug("Uploaded file \"" + fileObjKeyName + "\". The s3Url is: " + s3Url);
return s3Url; return s3Url;
} }
@ -207,6 +212,7 @@ public class S3ObjectStoreMinIO {
} }
} }
logger.info("Bucket " + bucketName + " was " + (shouldDeleteBucket ? "deleted!" : "emptied!"));
return true; return true;
} }
@ -222,5 +228,4 @@ public class S3ObjectStoreMinIO {
return true; return true;
} }
} }

View File

@ -21,6 +21,9 @@ spring.impala.url = jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/
# Note: The "UseNativeQuery" does not work with the PreparedStatements! Also, the url does not work without the ending "/" # Note: The "UseNativeQuery" does not work with the PreparedStatements! Also, the url does not work without the ending "/"
# The username and the password do not matter, since this app is always run in an pre-authenticated machine. # The username and the password do not matter, since this app is always run in an pre-authenticated machine.
spring.impala.oldDatabaseName = pdfaggregation_i
spring.impala.databaseName = pdfAggregationDatabase
spring.impala.driver-class-name = com.cloudera.impala.jdbc41.Driver spring.impala.driver-class-name = com.cloudera.impala.jdbc41.Driver
spring.datasource.hikari.pool-name=ControllerPool spring.datasource.hikari.pool-name=ControllerPool