From 95c38c4a2477cf3467e1b811dc73577464b0da89 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Wed, 7 Dec 2022 14:58:38 +0200 Subject: [PATCH] - Fix creating the "assignment" table, always in the testDatabase. - Code polishing. --- README.md | 2 +- .../urls_controller/configuration/ImpalaConnector.java | 4 ++-- src/main/resources/schemas/payload.avsc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5c0d1b7..c564c7a 100644 --- a/README.md +++ b/README.md @@ -9,5 +9,5 @@ To install and run the application: - Provide the **S3 Object Store** related configurations, inside the *src/main/resources/application.properties* file.
- Execute the ```installAndRun.sh``` script which builds and runs the app.
If you want to just run the app, then run the script with the argument "1": ```./installAndRun.sh 1```.
-If you want to build and run the app on a docker container, then run the script with the argument "0" followed by the argument "1": ```./installAndRun.sh 0 1```.
+If you want to build and run the app on a **Docker Container**, then run the script with the argument "0" followed by the argument "1": ```./installAndRun.sh 0 1```.

diff --git a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java index 7e9d78e..fe1e7a6 100644 --- a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java +++ b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java @@ -80,8 +80,8 @@ public class ImpalaConnector { // Drop the "current_assignment" table. It is a temporary table which is created on-demand during execution. jdbcTemplate.execute("DROP TABLE IF EXISTS " + databaseName + ".current_assignment PURGE"); - jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + testDatabaseName + ".assignment (id string, original_url string, workerid string, `date` bigint) stored as parquet"); - jdbcTemplate.execute("COMPUTE STATS " + testDatabaseName + ".assignment"); + jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".assignment (id string, original_url string, workerid string, `date` bigint) stored as parquet"); + jdbcTemplate.execute("COMPUTE STATS " + databaseName + ".assignment"); jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".attempt (id string, original_url string, `date` bigint, status string, error_class string, error_message string) stored as parquet"); jdbcTemplate.execute("COMPUTE STATS " + databaseName + ".attempt"); diff --git a/src/main/resources/schemas/payload.avsc b/src/main/resources/schemas/payload.avsc index a1a3180..07b90a3 100644 --- a/src/main/resources/schemas/payload.avsc +++ b/src/main/resources/schemas/payload.avsc @@ -5,7 +5,7 @@ "fields": [ {"name": "id", "type": "string"}, {"name": "original_url", "type": "string"}, - {"name": "actual_url", "type": "string"}, // This should not be null, since only the "found" pdf-publications are processed in parquet. + {"name": "actual_url", "type": "string"}, // This should NOT be null, since only the "found" pdf-publications are processed in parquet. {"name": "date", "type" : {"type": "long", "logicalType": "timestamp-millis"}}, {"name": "mimetype", "type": "string"}, {"name": "size", "type": ["null","string"]},