diff --git a/build.gradle b/build.gradle index e03648e..b8a2df8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { - id 'org.springframework.boot' version '2.7.5' - id 'io.spring.dependency-management' version '1.0.15.RELEASE' + id 'org.springframework.boot' version '2.7.6' + id 'io.spring.dependency-management' version '1.1.0' id 'java' } @@ -45,7 +45,7 @@ dependencies { // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' - implementation 'io.minio:minio:8.4.5' + implementation 'io.minio:minio:8.4.6' // https://mvnrepository.com/artifact/com.cloudera.impala/jdbc implementation("com.cloudera.impala:jdbc:2.5.31") { @@ -67,9 +67,6 @@ dependencies { // Avoid excluding 'org.apache.hive:hive-service', as this is needed and unfortunately, even adding a newer version separately, it introducing other vulnerable dependencies. } - // Add back some updated version of the needed dependencies. - implementation 'org.apache.thrift:libthrift:0.17.0' - // https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro implementation('org.apache.parquet:parquet-avro:1.12.3') @@ -85,6 +82,7 @@ dependencies { exclude group: 'com.google.protobuf', module: 'protobuf-java' exclude group: 'org.codehaus.jackson', module: 'jackson-core-asl' exclude group: 'org.codehaus.jackson', module: 'jackson-mapper-asl' + exclude group: 'com.fasterxml.woodstox', module: 'woodstox-core' //exclude group: 'commons-collections', module: 'commons-collections' // This dependency is required in order for the program to run without errors. } @@ -101,6 +99,10 @@ dependencies { exclude group: 'io.netty', module: 'netty' } + // Add back some updated version of the needed dependencies. + implementation 'org.apache.thrift:libthrift:0.17.0' + implementation 'com.fasterxml.woodstox:woodstox-core:6.4.0' + // https://mvnrepository.com/artifact/org.json/json implementation 'org.json:json:20220924' diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..9cc4b13 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,4 @@ +org.gradle.caching=true +org.gradle.parallel=true +org.gradle.caching.debug=false +org.gradle.warning.mode=all diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ae04661..f398c33 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/installAndRun.sh b/installAndRun.sh index 453d110..ed61024 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -1,4 +1,13 @@ -cd "${0%/*}" || (echo "Could not chdir to this script's dir!" && exit) # Change the working directory to the script's directory, when running from other location. +# This script installs and runs the project. + +# For error-handling, we cannot use the "set -e" since: it has problems https://mywiki.wooledge.org/BashFAQ/105 +# So we have our own function, for use when a single command fails. +handle_error () { + echo -e "$1"; exit $2 +} + +# Change the working directory to the script's directory, when running from another location. +cd "${0%/*}" || handle_error "Could not change-dir to this script's dir!" 1 justInstall=0 shouldRunInDocker=0 @@ -9,7 +18,7 @@ elif [[ $# -eq 2 ]]; then justInstall=$1 shouldRunInDocker=$2 elif [[ $# -gt 2 ]]; then - echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: script.sh "; exit 1 + echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: script.sh "; exit 2 fi if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then @@ -17,7 +26,7 @@ if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then justInstall=0 fi -gradleVersion="7.5.1" +gradleVersion="7.6" if [[ justInstall -eq 0 ]]; then @@ -42,7 +51,7 @@ if [[ justInstall -eq 0 ]]; then echo "Give the username for the Docker Hub:" read -r username echo -e "\nBuilding docker image..\n" - sudo docker --version || (echo -e "Docker was not found!"; exit 9) + sudo docker --version || handle_error "Docker was not found!" 3 dockerImage=${username}"/urls_controller:latest" sudo docker build -t "${dockerImage}" . echo -e "\nPushing docker image.. (the account password is required, otherwise it will not be pushed, but it will continue to run)..\n" diff --git a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java index a120bb5..7d1d493 100644 --- a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java @@ -522,16 +522,16 @@ public class ParquetFileUtils { // For each missing subdirectories, run the mkdirs-request. if ( !foundAttemptsDir ) { - logger.debug("The \"" + parquetHDFSDirectoryPathAttempts + "\" was not found! Going to create it."); + logger.debug("The remote parquet directory \"" + parquetHDFSDirectoryPathAttempts + "\" does not exist! Going to create it."); createHDFSDirectory(webHDFSBaseUrl + parquetHDFSDirectoryPathAttempts + mkdirsParams); } else - logger.info("The \"" + parquetHDFSDirectoryPathAttempts + "\" was found."); + logger.info("The remote parquet directory \"" + parquetHDFSDirectoryPathAttempts + "\" exists."); if ( !foundPayloadsDir ) { - logger.debug("The \"" + parquetHDFSDirectoryPathPayloads + "\" was not found! Going to create it."); + logger.debug("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloads + "\" does not exist! Going to create it."); createHDFSDirectory(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloads + mkdirsParams); } else - logger.info("The \"" + parquetHDFSDirectoryPathPayloads + "\" was found."); + logger.info("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloads + "\" exists."); } } catch (Exception e) { logger.error("", e);