- Improve some log-messages.
- Set some optimization settings for gradle. - Fix error-handling in "installAndRun.sh". - Update dependencies.
This commit is contained in:
parent
6226e2298d
commit
577ea983e8
14
build.gradle
14
build.gradle
|
@ -1,6 +1,6 @@
|
||||||
plugins {
|
plugins {
|
||||||
id 'org.springframework.boot' version '2.7.5'
|
id 'org.springframework.boot' version '2.7.6'
|
||||||
id 'io.spring.dependency-management' version '1.0.15.RELEASE'
|
id 'io.spring.dependency-management' version '1.1.0'
|
||||||
id 'java'
|
id 'java'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -45,7 +45,7 @@ dependencies {
|
||||||
// https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
|
// https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
|
||||||
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
|
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
|
||||||
|
|
||||||
implementation 'io.minio:minio:8.4.5'
|
implementation 'io.minio:minio:8.4.6'
|
||||||
|
|
||||||
// https://mvnrepository.com/artifact/com.cloudera.impala/jdbc
|
// https://mvnrepository.com/artifact/com.cloudera.impala/jdbc
|
||||||
implementation("com.cloudera.impala:jdbc:2.5.31") {
|
implementation("com.cloudera.impala:jdbc:2.5.31") {
|
||||||
|
@ -67,9 +67,6 @@ dependencies {
|
||||||
// Avoid excluding 'org.apache.hive:hive-service', as this is needed and unfortunately, even adding a newer version separately, it introducing other vulnerable dependencies.
|
// Avoid excluding 'org.apache.hive:hive-service', as this is needed and unfortunately, even adding a newer version separately, it introducing other vulnerable dependencies.
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add back some updated version of the needed dependencies.
|
|
||||||
implementation 'org.apache.thrift:libthrift:0.17.0'
|
|
||||||
|
|
||||||
// https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro
|
// https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro
|
||||||
implementation('org.apache.parquet:parquet-avro:1.12.3')
|
implementation('org.apache.parquet:parquet-avro:1.12.3')
|
||||||
|
|
||||||
|
@ -85,6 +82,7 @@ dependencies {
|
||||||
exclude group: 'com.google.protobuf', module: 'protobuf-java'
|
exclude group: 'com.google.protobuf', module: 'protobuf-java'
|
||||||
exclude group: 'org.codehaus.jackson', module: 'jackson-core-asl'
|
exclude group: 'org.codehaus.jackson', module: 'jackson-core-asl'
|
||||||
exclude group: 'org.codehaus.jackson', module: 'jackson-mapper-asl'
|
exclude group: 'org.codehaus.jackson', module: 'jackson-mapper-asl'
|
||||||
|
exclude group: 'com.fasterxml.woodstox', module: 'woodstox-core'
|
||||||
//exclude group: 'commons-collections', module: 'commons-collections' // This dependency is required in order for the program to run without errors.
|
//exclude group: 'commons-collections', module: 'commons-collections' // This dependency is required in order for the program to run without errors.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -101,6 +99,10 @@ dependencies {
|
||||||
exclude group: 'io.netty', module: 'netty'
|
exclude group: 'io.netty', module: 'netty'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add back some updated version of the needed dependencies.
|
||||||
|
implementation 'org.apache.thrift:libthrift:0.17.0'
|
||||||
|
implementation 'com.fasterxml.woodstox:woodstox-core:6.4.0'
|
||||||
|
|
||||||
// https://mvnrepository.com/artifact/org.json/json
|
// https://mvnrepository.com/artifact/org.json/json
|
||||||
implementation 'org.json:json:20220924'
|
implementation 'org.json:json:20220924'
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
org.gradle.caching=true
|
||||||
|
org.gradle.parallel=true
|
||||||
|
org.gradle.caching.debug=false
|
||||||
|
org.gradle.warning.mode=all
|
|
@ -1,5 +1,6 @@
|
||||||
distributionBase=GRADLE_USER_HOME
|
distributionBase=GRADLE_USER_HOME
|
||||||
distributionPath=wrapper/dists
|
distributionPath=wrapper/dists
|
||||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
|
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
|
||||||
|
networkTimeout=10000
|
||||||
zipStoreBase=GRADLE_USER_HOME
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
zipStorePath=wrapper/dists
|
zipStorePath=wrapper/dists
|
||||||
|
|
|
@ -1,4 +1,13 @@
|
||||||
cd "${0%/*}" || (echo "Could not chdir to this script's dir!" && exit) # Change the working directory to the script's directory, when running from other location.
|
# This script installs and runs the project.
|
||||||
|
|
||||||
|
# For error-handling, we cannot use the "set -e" since: it has problems https://mywiki.wooledge.org/BashFAQ/105
|
||||||
|
# So we have our own function, for use when a single command fails.
|
||||||
|
handle_error () {
|
||||||
|
echo -e "$1"; exit $2
|
||||||
|
}
|
||||||
|
|
||||||
|
# Change the working directory to the script's directory, when running from another location.
|
||||||
|
cd "${0%/*}" || handle_error "Could not change-dir to this script's dir!" 1
|
||||||
|
|
||||||
justInstall=0
|
justInstall=0
|
||||||
shouldRunInDocker=0
|
shouldRunInDocker=0
|
||||||
|
@ -9,7 +18,7 @@ elif [[ $# -eq 2 ]]; then
|
||||||
justInstall=$1
|
justInstall=$1
|
||||||
shouldRunInDocker=$2
|
shouldRunInDocker=$2
|
||||||
elif [[ $# -gt 2 ]]; then
|
elif [[ $# -gt 2 ]]; then
|
||||||
echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: script.sh <justInstall: 0 | 1> <shouldRunInDocker: 0 | 1>"; exit 1
|
echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: script.sh <justInstall: 0 | 1> <shouldRunInDocker: 0 | 1>"; exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then
|
if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then
|
||||||
|
@ -17,7 +26,7 @@ if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then
|
||||||
justInstall=0
|
justInstall=0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
gradleVersion="7.5.1"
|
gradleVersion="7.6"
|
||||||
|
|
||||||
if [[ justInstall -eq 0 ]]; then
|
if [[ justInstall -eq 0 ]]; then
|
||||||
|
|
||||||
|
@ -42,7 +51,7 @@ if [[ justInstall -eq 0 ]]; then
|
||||||
echo "Give the username for the Docker Hub:"
|
echo "Give the username for the Docker Hub:"
|
||||||
read -r username
|
read -r username
|
||||||
echo -e "\nBuilding docker image..\n"
|
echo -e "\nBuilding docker image..\n"
|
||||||
sudo docker --version || (echo -e "Docker was not found!"; exit 9)
|
sudo docker --version || handle_error "Docker was not found!" 3
|
||||||
dockerImage=${username}"/urls_controller:latest"
|
dockerImage=${username}"/urls_controller:latest"
|
||||||
sudo docker build -t "${dockerImage}" .
|
sudo docker build -t "${dockerImage}" .
|
||||||
echo -e "\nPushing docker image.. (the account password is required, otherwise it will not be pushed, but it will continue to run)..\n"
|
echo -e "\nPushing docker image.. (the account password is required, otherwise it will not be pushed, but it will continue to run)..\n"
|
||||||
|
|
|
@ -522,16 +522,16 @@ public class ParquetFileUtils {
|
||||||
|
|
||||||
// For each missing subdirectories, run the mkdirs-request.
|
// For each missing subdirectories, run the mkdirs-request.
|
||||||
if ( !foundAttemptsDir ) {
|
if ( !foundAttemptsDir ) {
|
||||||
logger.debug("The \"" + parquetHDFSDirectoryPathAttempts + "\" was not found! Going to create it.");
|
logger.debug("The remote parquet directory \"" + parquetHDFSDirectoryPathAttempts + "\" does not exist! Going to create it.");
|
||||||
createHDFSDirectory(webHDFSBaseUrl + parquetHDFSDirectoryPathAttempts + mkdirsParams);
|
createHDFSDirectory(webHDFSBaseUrl + parquetHDFSDirectoryPathAttempts + mkdirsParams);
|
||||||
} else
|
} else
|
||||||
logger.info("The \"" + parquetHDFSDirectoryPathAttempts + "\" was found.");
|
logger.info("The remote parquet directory \"" + parquetHDFSDirectoryPathAttempts + "\" exists.");
|
||||||
|
|
||||||
if ( !foundPayloadsDir ) {
|
if ( !foundPayloadsDir ) {
|
||||||
logger.debug("The \"" + parquetHDFSDirectoryPathPayloads + "\" was not found! Going to create it.");
|
logger.debug("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloads + "\" does not exist! Going to create it.");
|
||||||
createHDFSDirectory(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloads + mkdirsParams);
|
createHDFSDirectory(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloads + mkdirsParams);
|
||||||
} else
|
} else
|
||||||
logger.info("The \"" + parquetHDFSDirectoryPathPayloads + "\" was found.");
|
logger.info("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloads + "\" exists.");
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
|
|
Loading…
Reference in New Issue