From e975bec911b051a25570a00ed7cf155c598e8eca Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Tue, 21 Mar 2023 16:46:33 +0200 Subject: [PATCH] - Add Prometheus and Grafana which help measuring various metrics for the Controller's health and performance. - Fix Docker config still using the old (now removed) "application.properties" file. - Simplify the process of building and running the docker image; Now we use docker compose to run the Controller, along with the Prometheus and Grafana. Also, now it is not requested from the user to login and push the image (this may change in the future). --- Dockerfile | 2 +- build.gradle | 6 +++++ docker-compose.yml | 39 ++++++++++++++++++++++++++++++ installAndRun.sh | 15 ++++-------- src/main/resources/application.yml | 25 +++++++++++++++++-- src/main/resources/prometheus.yml | 18 ++++++++++++++ 6 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 docker-compose.yml create mode 100644 src/main/resources/prometheus.yml diff --git a/Dockerfile b/Dockerfile index fc501d6..88a5d21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,4 @@ COPY build/libs/*-SNAPSHOT.jar urls_controller.jar EXPOSE 1880 -ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.properties"] +ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.yml"] diff --git a/build.gradle b/build.gradle index f341534..f4de997 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,9 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-jdbc") implementation("org.springframework.boot:spring-boot-configuration-processor") + implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("org.springframework.boot:spring-boot-starter-aop") + implementation("org.springframework.security:spring-security-core") implementation("org.springframework.security:spring-security-web") implementation("org.springframework.security:spring-security-config") @@ -109,6 +112,9 @@ dependencies { // https://mvnrepository.com/artifact/org.json/json implementation 'org.json:json:20230227' + // https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus + runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.10.5' + testImplementation 'org.springframework.security:spring-security-test' testImplementation "org.springframework.boot:spring-boot-starter-test" } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..84b961a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3.3' + +services: + + urls_controller: + image: 'pdf_aggregation_service/urls_controller:latest' + ports: + - '1880:1880' + volumes: + - type: bind + source: $HOME/tmp/config + target: /mnt/config + - type: bind + source: $HOME/tmp + target: /tmp + - type: bind + source: $HOME/logs + target: /logs + build: + dockerfile: ./Dockerfile + context: . + + prometheus: + image: 'prom/prometheus:latest' + ports: + - '9090:9090' + command: '--config.file=/etc/prometheus/config.yml' + volumes: + - './src/main/resources/prometheus.yml:/etc/prometheus/config.yml' + depends_on: + - urls_controller + + grafana: + image: 'grafana/grafana:latest' + ports: + - '3000:3000' + depends_on: + - urls_controller + - prometheus diff --git a/installAndRun.sh b/installAndRun.sh index f5c5251..8e70226 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -48,17 +48,12 @@ if [[ justInstall -eq 0 ]]; then gradle clean build if [[ shouldRunInDocker -eq 1 ]]; then - echo -e "\nGive the username for the Docker Hub:" - read -r username - echo -e "\nBuilding docker image..\n" + echo -e "\nBuilding the docker image and running the containers..\n" sudo docker --version || handle_error "Docker was not found!" 3 - dockerImage=${username}"/urls_controller:latest" - sudo docker build --no-cache -t "${dockerImage}" . - echo -e "\nPushing docker image.. (the account password is required, otherwise it will not be pushed, but it will continue to run)..\n" - (sudo docker login -u "${username}" && sudo docker push "${dockerImage}") || true - (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true # This also replaces an existing "application.properties". - sudo docker run -d --mount type=bind,source="$HOME"/tmp/config,target=/mnt/config -p 1880:1880 "${dockerImage}" && echo "The docker container started running." - # Run in "detached mode" (in the background). + (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.yml "$HOME"/tmp/config) || true # This also replaces an existing "application.yml". + sudo mkdir -p "$HOME"/logs || true + sudo docker compose up --build -d && echo "The Urls_Controller, Prometheus and Grafana docker-containers started running." + # Run in "detached mode" -d (in the background). fi else export PATH=/opt/gradle/gradle-${gradleVersion}/bin:$PATH # Make sure the gradle is still accessible (it usually isn't without the "export"). diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 67a7032..ea6413a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -30,13 +30,15 @@ services: shouldShowAllS3Buckets: true datasources: # Provide a list of datasource IDs, which should be excluded from crawling. Their content is either bulk-imported or is known to be restricted. - excludedIDs: > # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists. + excludedIDs: > # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists (at least without additional config). opendoar____::6f4922f45568161a8cdf4ad2299f6d23 # Since we use a multi-line value from our list, we add the ID-explanations here (otherwise comments will be part of values): # First-id: arXiv.org e-Print Archive spring: + application: + name: Urls_Controller datasource: driver-class-name: com.cloudera.impala.jdbc41.Driver url: XA @@ -63,7 +65,26 @@ hdfs: # Or by giving the username AND the password, in order for the program to crete the auth-String programmatically. # The first approach is intended for more privacy, while the second for more ease. Either way, all three should be uncommented, no matter which ones are used. - parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/ + parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/ # In case the "isTestEnvironment" is "true", this is automatically supplemented by a "/test/" subdirectory, to avoid any conflicts. + +# Prometheus related config. +management: + endpoint: + health: + enabled: true + show-details: always + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + base-path: /actuator + exposure: + include: health,info,prometheus,metrics + metrics: + tags: + application: ${spring.application.name} logging: level: diff --git a/src/main/resources/prometheus.yml b/src/main/resources/prometheus.yml new file mode 100644 index 0000000..72101e3 --- /dev/null +++ b/src/main/resources/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s # Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + +# A scrape configuration containing exactly one endpoint to scrape: +scrape_configs: + - job_name: 'prometheus' # Job to scrape Prometheus metrics + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'spring-actuator' + metrics_path: '/api/actuator/prometheus' # Job to scrape application metrics + scrape_interval: 15s + scrape_timeout: 10s + static_configs: # TODO - Find a way to automatically apply the publicIP of the host machine it is running on. + - targets: [ ':1880' ]