- Add Prometheus and Grafana which help measuring various metrics for the Controller's health and performance.

- Fix Docker config still using the old (now removed) "application.properties" file.
- Simplify the process of building and running the docker image; Now we use docker compose to run the Controller, along with the Prometheus and Grafana. Also, now it is not requested from the user to login and push the image (this may change in the future).
This commit is contained in:
Lampros Smyrnaios 2023-03-21 16:46:33 +02:00
parent 003c0bf179
commit e975bec911
6 changed files with 92 additions and 13 deletions

View File

@ -4,4 +4,4 @@ COPY build/libs/*-SNAPSHOT.jar urls_controller.jar
EXPOSE 1880
ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.properties"]
ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.yml"]

View File

@ -28,6 +28,9 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-jdbc")
implementation("org.springframework.boot:spring-boot-configuration-processor")
implementation("org.springframework.boot:spring-boot-starter-actuator")
implementation("org.springframework.boot:spring-boot-starter-aop")
implementation("org.springframework.security:spring-security-core")
implementation("org.springframework.security:spring-security-web")
implementation("org.springframework.security:spring-security-config")
@ -109,6 +112,9 @@ dependencies {
// https://mvnrepository.com/artifact/org.json/json
implementation 'org.json:json:20230227'
// https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.10.5'
testImplementation 'org.springframework.security:spring-security-test'
testImplementation "org.springframework.boot:spring-boot-starter-test"
}

39
docker-compose.yml Normal file
View File

@ -0,0 +1,39 @@
version: '3.3'
services:
urls_controller:
image: 'pdf_aggregation_service/urls_controller:latest'
ports:
- '1880:1880'
volumes:
- type: bind
source: $HOME/tmp/config
target: /mnt/config
- type: bind
source: $HOME/tmp
target: /tmp
- type: bind
source: $HOME/logs
target: /logs
build:
dockerfile: ./Dockerfile
context: .
prometheus:
image: 'prom/prometheus:latest'
ports:
- '9090:9090'
command: '--config.file=/etc/prometheus/config.yml'
volumes:
- './src/main/resources/prometheus.yml:/etc/prometheus/config.yml'
depends_on:
- urls_controller
grafana:
image: 'grafana/grafana:latest'
ports:
- '3000:3000'
depends_on:
- urls_controller
- prometheus

View File

@ -48,17 +48,12 @@ if [[ justInstall -eq 0 ]]; then
gradle clean build
if [[ shouldRunInDocker -eq 1 ]]; then
echo -e "\nGive the username for the Docker Hub:"
read -r username
echo -e "\nBuilding docker image..\n"
echo -e "\nBuilding the docker image and running the containers..\n"
sudo docker --version || handle_error "Docker was not found!" 3
dockerImage=${username}"/urls_controller:latest"
sudo docker build --no-cache -t "${dockerImage}" .
echo -e "\nPushing docker image.. (the account password is required, otherwise it will not be pushed, but it will continue to run)..\n"
(sudo docker login -u "${username}" && sudo docker push "${dockerImage}") || true
(sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true # This also replaces an existing "application.properties".
sudo docker run -d --mount type=bind,source="$HOME"/tmp/config,target=/mnt/config -p 1880:1880 "${dockerImage}" && echo "The docker container started running."
# Run in "detached mode" (in the background).
(sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.yml "$HOME"/tmp/config) || true # This also replaces an existing "application.yml".
sudo mkdir -p "$HOME"/logs || true
sudo docker compose up --build -d && echo "The Urls_Controller, Prometheus and Grafana docker-containers started running."
# Run in "detached mode" -d (in the background).
fi
else
export PATH=/opt/gradle/gradle-${gradleVersion}/bin:$PATH # Make sure the gradle is still accessible (it usually isn't without the "export").

View File

@ -30,13 +30,15 @@ services:
shouldShowAllS3Buckets: true
datasources: # Provide a list of datasource IDs, which should be excluded from crawling. Their content is either bulk-imported or is known to be restricted.
excludedIDs: > # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists.
excludedIDs: > # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists (at least without additional config).
opendoar____::6f4922f45568161a8cdf4ad2299f6d23
# Since we use a multi-line value from our list, we add the ID-explanations here (otherwise comments will be part of values):
# First-id: arXiv.org e-Print Archive
spring:
application:
name: Urls_Controller
datasource:
driver-class-name: com.cloudera.impala.jdbc41.Driver
url: XA
@ -63,7 +65,26 @@ hdfs:
# Or by giving the username AND the password, in order for the program to crete the auth-String programmatically.
# The first approach is intended for more privacy, while the second for more ease. Either way, all three should be uncommented, no matter which ones are used.
parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/
parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/ # In case the "isTestEnvironment" is "true", this is automatically supplemented by a "/test/" subdirectory, to avoid any conflicts.
# Prometheus related config.
management:
endpoint:
health:
enabled: true
show-details: always
metrics:
enabled: true
prometheus:
enabled: true
endpoints:
web:
base-path: /actuator
exposure:
include: health,info,prometheus,metrics
metrics:
tags:
application: ${spring.application.name}
logging:
level:

View File

@ -0,0 +1,18 @@
global:
scrape_interval: 15s # Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# A scrape configuration containing exactly one endpoint to scrape:
scrape_configs:
- job_name: 'prometheus' # Job to scrape Prometheus metrics
scrape_interval: 15s
scrape_timeout: 10s
static_configs:
- targets: ['localhost:9090']
- job_name: 'spring-actuator'
metrics_path: '/api/actuator/prometheus' # Job to scrape application metrics
scrape_interval: 15s
scrape_timeout: 10s
static_configs: # TODO - Find a way to automatically apply the publicIP of the host machine it is running on.
- targets: [ '<SERVER_IP>:1880' ]