- Add Prometheus and Grafana which help measuring various metrics for the Controller's health and performance.

- Fix Docker config still using the old (now removed) "application.properties" file.
- Simplify the process of building and running the docker image; Now we use docker compose to run the Controller, along with the Prometheus and Grafana. Also, now it is not requested from the user to login and push the image (this may change in the future).

Dockerfile

@@ -4,4 +4,4 @@ COPY build/libs/*-SNAPSHOT.jar urls_controller.jar
 
 EXPOSE 1880
 
-ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.properties"]
+ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.yml"]

build.gradle

@@ -28,6 +28,9 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-jdbc")
 
     implementation("org.springframework.boot:spring-boot-configuration-processor")
+    implementation("org.springframework.boot:spring-boot-starter-actuator")
+    implementation("org.springframework.boot:spring-boot-starter-aop")
+
     implementation("org.springframework.security:spring-security-core")
     implementation("org.springframework.security:spring-security-web")
     implementation("org.springframework.security:spring-security-config")
@@ -109,6 +112,9 @@ dependencies {
     // https://mvnrepository.com/artifact/org.json/json
     implementation 'org.json:json:20230227'
 
+    // https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus
+    runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.10.5'
+
     testImplementation 'org.springframework.security:spring-security-test'
     testImplementation "org.springframework.boot:spring-boot-starter-test"
 }

docker-compose.yml

@@ -0,0 +1,39 @@
+version: '3.3'
+
+services:
+
+  urls_controller:
+    image: 'pdf_aggregation_service/urls_controller:latest'
+    ports:
+      - '1880:1880'
+    volumes:
+      - type: bind
+        source: $HOME/tmp/config
+        target: /mnt/config
+      - type: bind
+        source: $HOME/tmp
+        target: /tmp
+      - type: bind
+        source: $HOME/logs
+        target: /logs
+    build:
+      dockerfile: ./Dockerfile
+      context: .
+
+  prometheus:
+    image: 'prom/prometheus:latest'
+    ports:
+      - '9090:9090'
+    command: '--config.file=/etc/prometheus/config.yml'
+    volumes:
+      - './src/main/resources/prometheus.yml:/etc/prometheus/config.yml'
+    depends_on:
+      - urls_controller
+
+  grafana:
+    image: 'grafana/grafana:latest'
+    ports:
+      - '3000:3000'
+    depends_on:
+      - urls_controller
+      - prometheus

installAndRun.sh

@@ -48,17 +48,12 @@ if [[ justInstall -eq 0 ]]; then
   gradle clean build
 
   if [[ shouldRunInDocker -eq 1 ]]; then
-    echo -e "\nGive the username for the Docker Hub:"
-    read -r username
-    echo -e "\nBuilding docker image..\n"
+    echo -e "\nBuilding the docker image and running the containers..\n"
     sudo docker --version || handle_error "Docker was not found!" 3
-    dockerImage=${username}"/urls_controller:latest"
-    sudo docker build --no-cache -t "${dockerImage}" .
-    echo -e "\nPushing docker image.. (the account password is required, otherwise it will not be pushed, but it will continue to run)..\n"
-    (sudo docker login -u "${username}" && sudo docker push "${dockerImage}") || true
-    (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true  # This also replaces an existing "application.properties".
-    sudo docker run -d --mount type=bind,source="$HOME"/tmp/config,target=/mnt/config -p 1880:1880 "${dockerImage}" && echo "The docker container started running."
-    # Run in "detached mode" (in the background).
+    (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.yml "$HOME"/tmp/config) || true  # This also replaces an existing "application.yml".
+    sudo mkdir -p "$HOME"/logs || true
+    sudo docker compose up --build -d && echo "The Urls_Controller, Prometheus and Grafana docker-containers started running."
+    # Run in "detached mode" -d (in the background).
   fi
 else
   export PATH=/opt/gradle/gradle-${gradleVersion}/bin:$PATH # Make sure the gradle is still accessible (it usually isn't without the "export").

src/main/resources/application.yml

@@ -30,13 +30,15 @@ services:
                 shouldShowAllS3Buckets: true
 
             datasources:    # Provide a list of datasource IDs, which should be excluded from crawling. Their content is either bulk-imported or is known to be restricted.
-                excludedIDs: >  # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists.
+                excludedIDs: >  # Use comma-seperated values (one in each line for best readability), as Spring has is currently incapable of parsing Dropwizard-styled lists (at least without additional config).
                     opendoar____::6f4922f45568161a8cdf4ad2299f6d23
 
                 # Since we use a multi-line value from our list, we add the ID-explanations here (otherwise comments will be part of values):
                 # First-id: arXiv.org e-Print Archive
 
 spring:
+    application:
+        name: Urls_Controller
     datasource:
         driver-class-name: com.cloudera.impala.jdbc41.Driver
         url: XA
@@ -63,7 +65,26 @@ hdfs:
     # Or by giving the username AND the password, in order for the program to crete the auth-String programmatically.
     # The first approach is intended for more privacy, while the second for more ease. Either way, all three should be uncommented, no matter which ones are used.
 
-    parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/
+    parquetRemoteBaseDirectoryPath: /tmp/parquet_uploads/   # In case the "isTestEnvironment" is "true", this is automatically supplemented by a "/test/" subdirectory, to avoid any conflicts.
+
+# Prometheus related config.
+management:
+    endpoint:
+        health:
+            enabled: true
+            show-details: always
+        metrics:
+            enabled: true
+        prometheus:
+            enabled: true
+    endpoints:
+        web:
+            base-path: /actuator
+            exposure:
+                include: health,info,prometheus,metrics
+    metrics:
+        tags:
+            application: ${spring.application.name}
 
 logging:
     level:

src/main/resources/prometheus.yml

@@ -0,0 +1,18 @@
+global:
+  scrape_interval:     15s # Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+
+# A scrape configuration containing exactly one endpoint to scrape:
+scrape_configs:
+  - job_name: 'prometheus' # Job to scrape Prometheus metrics
+    scrape_interval: 15s
+    scrape_timeout: 10s
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'spring-actuator'
+    metrics_path: '/api/actuator/prometheus' # Job to scrape application metrics
+    scrape_interval: 15s
+    scrape_timeout: 10s
+    static_configs: # TODO - Find a way to automatically apply the publicIP of the host machine it is running on.
+      - targets: [ '<SERVER_IP>:1880' ]