From 4280f89296212d988ced6d425e36a1c143bdac0a Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Tue, 21 Mar 2023 17:04:28 +0200 Subject: [PATCH] - Set the default value of the "isTestEnvironment" property to "true", in order to avoid undesired outcomes in the production db. - Code polishing. --- .../openaire/urls_controller/components/ScheduledTasks.java | 2 +- .../urls_controller/configuration/ImpalaConnector.java | 3 ++- .../openaire/urls_controller/services/UrlsServiceImpl.java | 6 ++---- .../java/eu/openaire/urls_controller/util/FileUtils.java | 2 +- .../eu/openaire/urls_controller/util/ParquetFileUtils.java | 2 +- .../java/eu/openaire/urls_controller/util/UriBuilder.java | 4 ++-- src/main/resources/application.yml | 2 +- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java index f0b4ead..e1686fd 100644 --- a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java @@ -20,4 +20,4 @@ public class ScheduledTasks { public void reportCurrentTime() { logger.info("Server is live! Time is now {}", dateFormat.format(new Date())); } -} \ No newline at end of file +} diff --git a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java index 97e72ff..751461c 100644 --- a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java +++ b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java @@ -69,9 +69,10 @@ public class ImpalaConnector { jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + testDatabaseName + ".datasource stored as parquet as select * from " + initialDatabaseName + ".datasource"); jdbcTemplate.execute("COMPUTE STATS " + testDatabaseName + ".datasource"); - databaseName = testDatabaseName; // For the rest of the queries. + databaseName = testDatabaseName; } else { logger.info("Going to create or validate the tables that are populated by the Controller, for the \"initialDatabase\" = \"" + initialDatabaseName + "\""); + // Note that for the "initialDatabase", the initial 4 tables are expected to be created either manually or by other pieces of software, as views of the contents of the Graph. databaseName = initialDatabaseName; } diff --git a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java index 029b979..7620bdf 100644 --- a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java +++ b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java @@ -40,11 +40,9 @@ public class UrlsServiceImpl implements UrlsService { @Autowired private JdbcTemplate jdbcTemplate; - @Autowired private FileUtils fileUtils; - @Autowired private ParquetFileUtils parquetFileUtils; @@ -75,12 +73,12 @@ public class UrlsServiceImpl implements UrlsService { for ( int i=0; i < exclusionListSize; ++i ) { sb.append("\"").append(excludedIDs.get(i)).append("\""); if ( i < (exclusionListSize -1) ) - sb.append(","); + sb.append(", "); } sb.append(")"); excludedDatasourceIDsStringList = sb.toString(); - //logger.debug("excludedDatasourceIDsStringList :\n" + excludedDatasourceIDsStringList); // DEBUG! + logger.info("The following datasources will be excluded from crawling: " + excludedDatasourceIDsStringList); } diff --git a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java index b7b1a5f..f7196bb 100644 --- a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java @@ -240,7 +240,7 @@ public class FileUtils { String baseUrl = "http://" + remoteAddr + ":1881/api/full-texts/getFullTexts/" + assignmentsBatchCounter + "/" + numOfBatches + "/"; // TODO - The worker should send the port in which it accepts requests, along with the current request. - // TODO - The least we have to do it to expose the port-assignment somewhere more obvious like inside the "application.properties" file. + // TODO - The least we have to do it to expose the port-assignment somewhere more obvious like inside the "application.yml" file. String curAssignmentsBaseLocation = baseFilesLocation + "assignments_" + assignmentsBatchCounter + File.separator; // Note: the "curAssignmentsBaseLocation"-directory will be created once the first batch subdirectory is called for creation. diff --git a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java index 09d2663..8fe3d9c 100644 --- a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java @@ -631,7 +631,7 @@ public class ParquetFileUtils { int statusCode = conn.getResponseCode(); if ( statusCode == 200 ) { - logger.debug("The file \"" + parquetFileName + "\" was successfully deleted."); // DEBUG! + logger.debug("The file \"" + parquetFileName + "\" was successfully deleted."); } else { String errorMsg = "The file \"" + parquetFileName + "\" could not be deleted! Response-code: " + statusCode; logger.error(errorMsg); diff --git a/src/main/java/eu/openaire/urls_controller/util/UriBuilder.java b/src/main/java/eu/openaire/urls_controller/util/UriBuilder.java index 4bf59c7..8a076f7 100644 --- a/src/main/java/eu/openaire/urls_controller/util/UriBuilder.java +++ b/src/main/java/eu/openaire/urls_controller/util/UriBuilder.java @@ -25,7 +25,7 @@ public class UriBuilder { String sslEnabled = environment.getProperty("server.ssl.enabled"); if (sslEnabled == null) { // It's expected to not exist if there is no SSL-configuration. - logger.warn("No property \"server.ssl.enabled\" was found in \"application.properties\". Continuing with plain HTTP.."); + logger.warn("No property \"server.ssl.enabled\" was found in \"application.yml\". Continuing with plain HTTP.."); sslEnabled = "false"; } baseUrl += sslEnabled.equals("true") ? "s" : ""; @@ -45,7 +45,7 @@ public class UriBuilder { if ( !baseInternalPath.endsWith("/") ) baseUrl += "/"; } else { - logger.warn("No property \"server.servlet.context-path\" was found in \"application.properties\"!"); // Yes it's expected. + logger.warn("No property \"server.servlet.context-path\" was found in \"application.yml\"!"); // Yes it's expected. baseUrl += "/"; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index ea6413a..3a99a88 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -6,7 +6,7 @@ server: services: pdfaggregation: controller: - isTestEnvironment: false + isTestEnvironment: true # In case the "isTestEnvironment" is "true", the "testDatabase" below and all its tables are created (if not exist). # The tables "datasource", "publication", "publication_pids" and "publication_urls" are filled with the data from the same tables existing in the "initialDatabase", if they don't exist. # In case the "isTestEnvironment" is "false", the "initialDatabase" is used. The Controller assumes that the above 4 tables are present, and only creates, if they don't exist, the following tables: