diff --git a/Dockerfile b/Dockerfile index fc501d6..88a5d21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,4 @@ COPY build/libs/*-SNAPSHOT.jar urls_controller.jar EXPOSE 1880 -ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.properties"] +ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.yml"] diff --git a/build.gradle b/build.gradle index f341534..f4de997 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,9 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-jdbc") implementation("org.springframework.boot:spring-boot-configuration-processor") + implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("org.springframework.boot:spring-boot-starter-aop") + implementation("org.springframework.security:spring-security-core") implementation("org.springframework.security:spring-security-web") implementation("org.springframework.security:spring-security-config") @@ -109,6 +112,9 @@ dependencies { // https://mvnrepository.com/artifact/org.json/json implementation 'org.json:json:20230227' + // https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus + runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.10.5' + testImplementation 'org.springframework.security:spring-security-test' testImplementation "org.springframework.boot:spring-boot-starter-test" } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..84b961a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3.3' + +services: + + urls_controller: + image: 'pdf_aggregation_service/urls_controller:latest' + ports: + - '1880:1880' + volumes: + - type: bind + source: $HOME/tmp/config + target: /mnt/config + - type: bind + source: $HOME/tmp + target: /tmp + - type: bind + source: $HOME/logs + target: /logs + build: + dockerfile: ./Dockerfile + context: . + + prometheus: + image: 'prom/prometheus:latest' + ports: + - '9090:9090' + command: '--config.file=/etc/prometheus/config.yml' + volumes: + - './src/main/resources/prometheus.yml:/etc/prometheus/config.yml' + depends_on: + - urls_controller + + grafana: + image: 'grafana/grafana:latest' + ports: + - '3000:3000' + depends_on: + - urls_controller + - prometheus diff --git a/installAndRun.sh b/installAndRun.sh index f5c5251..8e70226 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -48,17 +48,12 @@ if [[ justInstall -eq 0 ]]; then gradle clean build if [[ shouldRunInDocker -eq 1 ]]; then - echo -e "\nGive the username for the Docker Hub:" - read -r username - echo -e "\nBuilding docker image..\n" + echo -e "\nBuilding the docker image and running the containers..\n" sudo docker --version || handle_error "Docker was not found!" 3 - dockerImage=${username}"/urls_controller:latest" - sudo docker build --no-cache -t "${dockerImage}" . - echo -e "\nPushing docker image.. (the account password is required, otherwise it will not be pushed, but it will continue to run)..\n" - (sudo docker login -u "${username}" && sudo docker push "${dockerImage}") || true - (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true # This also replaces an existing "application.properties". - sudo docker run -d --mount type=bind,source="$HOME"/tmp/config,target=/mnt/config -p 1880:1880 "${dockerImage}" && echo "The docker container started running." - # Run in "detached mode" (in the background). + (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.yml "$HOME"/tmp/config) || true # This also replaces an existing "application.yml". + sudo mkdir -p "$HOME"/logs || true + sudo docker compose up --build -d && echo "The Urls_Controller, Prometheus and Grafana docker-containers started running." + # Run in "detached mode" -d (in the background). fi else export PATH=/opt/gradle/gradle-${gradleVersion}/bin:$PATH # Make sure the gradle is still accessible (it usually isn't without the "export"). diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 67a7032..ea6413a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -30,13 +30,15 @@ services: shouldShowAllS3Buckets: true datasources: # Provide a list of datasource IDs, which should be excluded from crawling. Their content is either bulk-imported or is known to be restricted. - excludedIDs: > # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists. + excludedIDs: > # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists (at least without additional config). opendoar____::6f4922f45568161a8cdf4ad2299f6d23 # Since we use a multi-line value from our list, we add the ID-explanations here (otherwise comments will be part of values): # First-id: arXiv.org e-Print Archive spring: + application: + name: Urls_Controller datasource: driver-class-name: com.cloudera.impala.jdbc41.Driver url: XA @@ -63,7 +65,26 @@ hdfs: # Or by giving the username AND the password, in order for the program to crete the auth-String programmatically. # The first approach is intended for more privacy, while the second for more ease. Either way, all three should be uncommented, no matter which ones are used. - parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/ + parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/ # In case the "isTestEnvironment" is "true", this is automatically supplemented by a "/test/" subdirectory, to avoid any conflicts. + +# Prometheus related config. +management: + endpoint: + health: + enabled: true + show-details: always + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + base-path: /actuator + exposure: + include: health,info,prometheus,metrics + metrics: + tags: + application: ${spring.application.name} logging: level: diff --git a/src/main/resources/prometheus.yml b/src/main/resources/prometheus.yml new file mode 100644 index 0000000..72101e3 --- /dev/null +++ b/src/main/resources/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s # Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + +# A scrape configuration containing exactly one endpoint to scrape: +scrape_configs: + - job_name: 'prometheus' # Job to scrape Prometheus metrics + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'spring-actuator' + metrics_path: '/api/actuator/prometheus' # Job to scrape application metrics + scrape_interval: 15s + scrape_timeout: 10s + static_configs: # TODO - Find a way to automatically apply the publicIP of the host machine it is running on. + - targets: [ ':1880' ]