Initial commit of UrlsWorker.

build.gradle

@@ -0,0 +1,49 @@
+buildscript {
+    ext {
+        springSecurityVersion = "5.4.5"
+    }
+}
+
+plugins {
+    id 'org.springframework.boot' version '2.4.3'
+    id 'io.spring.dependency-management' version '1.0.11.RELEASE'
+    id 'java'
+}
+
+group = 'eu.openaire.urls_worker'
+version = '0.0.1-SNAPSHOT'
+sourceCompatibility = '1.8'
+
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    runtimeOnly 'org.springframework.boot:spring-boot-devtools'
+
+    implementation 'org.springframework.boot:spring-boot-starter-web'
+    implementation("org.springframework.boot:spring-boot-starter-security")
+    implementation("org.springframework.boot:spring-boot-configuration-processor")
+    implementation("org.springframework.security:spring-security-core:${springSecurityVersion}")
+    implementation("org.springframework.security:spring-security-web:${springSecurityVersion}")
+    implementation("org.springframework.security:spring-security-config:${springSecurityVersion}")
+    implementation("io.jsonwebtoken:jjwt:0.9.1")
+
+
+    implementation "org.projectlombok:lombok:1.18.18"
+    implementation 'com.google.code.gson:gson:2.8.6'
+    implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
+    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
+    implementation group: 'commons-io', name: 'commons-io', version: '2.8.0'
+
+    // https://mvnrepository.com/artifact/com.google.guava/guava
+    // implementation group: 'com.google.guava', name: 'guava', version: '30.1-jre' // It will be usefull later..
+
+    testImplementation group: 'org.springframework.security', name: 'spring-security-test', version: springSecurityVersion
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
+}
+
+test {
+    useJUnitPlatform()
+}

scripts/startServer.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -x
+cd ../
+./gradlew bootrun

settings.gradle

@@ -0,0 +1,6 @@
+pluginManagement {
+    repositories {
+        gradlePluginPortal()
+    }
+}
+rootProject.name = 'urls_worker'

src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java

@@ -0,0 +1,13 @@
+package eu.openaire.urls_worker;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class UrlsWorkerApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(UrlsWorkerApplication.class, args);
+    }
+
+}

src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java

@@ -0,0 +1,23 @@
+package eu.openaire.urls_worker.components;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+
+@Component
+public class ScheduledTasks {
+
+    private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
+
+    private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
+    @Scheduled(fixedRate = 600_000)   // TODO - Change to every 10 mins: 600_000
+    public void reportCurrentTime() {
+        logger.info("Server is live! Time is now {}", dateFormat.format(new Date()));
+    }
+}

src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java

@@ -0,0 +1,27 @@
+package eu.openaire.urls_worker.controllers;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+
+@RestController
+@RequestMapping("")
+public class GeneralController {
+
+    private static final Logger logger = LoggerFactory.getLogger(GeneralController.class);
+
+
+    public GeneralController() {
+    }
+
+    @GetMapping("isAlive")
+    public ResponseEntity<?> isWorkerAlive() {
+
+        logger.info("Received an \"isAlive\" request.");
+
+        return ResponseEntity.ok().build();
+    }
+
+}

src/main/java/eu/openaire/urls_worker/models/Payload.java

@@ -0,0 +1,150 @@
+package eu.openaire.urls_worker.models;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({
+        "id",
+        "original_url",
+        "actual_url",
+        "date_acquired",
+        "mime_type",
+        "size",
+        "more_info",
+        "md5",
+        "location",
+        "provenance"
+})
+public class Payload {
+
+    @JsonProperty("id")
+    private String id;
+
+    @JsonProperty("original_url")
+    private String original_url;
+
+    @JsonProperty("actual_url")
+    private String actual_url;
+
+    @JsonProperty("date_acquired")
+    private String date_acquired;
+
+    @JsonProperty("mime_type")
+    private String mime_type;
+
+    @JsonProperty("size")
+    private String size;
+
+    @JsonProperty("more_info")
+    private  String more_info;
+
+    @JsonProperty("md5")
+    private String md5;
+
+    @JsonProperty("location")
+    private String location;
+
+    @JsonProperty("provenance")
+    private String provenance;
+
+
+    public String getOriginal_url() {
+        return original_url;
+    }
+
+    public void setOriginal_url(String original_url) {
+        this.original_url = original_url;
+    }
+
+    public String getActual_url() {
+        return actual_url;
+    }
+
+    public void setActual_url(String actual_url) {
+        this.actual_url = actual_url;
+    }
+
+    public String getDate_acquired() {
+        return date_acquired;
+    }
+
+    public void setDate_acquired(String date_acquired) {
+        this.date_acquired = date_acquired;
+    }
+
+    public String getMime_type() {
+        return mime_type;
+    }
+
+    public void setMime_type(String mime_type) {
+        this.mime_type = mime_type;
+    }
+
+    public String getSize() {
+        return size;
+    }
+
+    public void setSize(String size) {
+        this.size = size;
+    }
+
+    public String getMore_info() {
+        return more_info;
+    }
+
+    public void setMore_info(String more_info) {
+        this.more_info = more_info;
+    }
+
+    public String getMd5() {
+        return md5;
+    }
+
+    public void setMd5(String md5) {
+        this.md5 = md5;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    public void setLocation(String location) {
+        this.location = location;
+    }
+
+    public String getProvenance() {
+        return provenance;
+    }
+
+    public void setProvenance(String provenance) {
+        this.provenance = provenance;
+    }
+
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    @Override
+    public String toString() {
+        return "Payload{" +
+                "id='" + id + '\'' +
+                ", original_url='" + original_url + '\'' +
+                ", actual_url='" + actual_url + '\'' +
+                ", date_acquired='" + date_acquired + '\'' +
+                ", mime_type='" + mime_type + '\'' +
+                ", size='" + size + '\'' +
+                ", more_info='" + more_info + '\'' +
+                ", md5='" + md5 + '\'' +
+                ", location='" + location + '\'' +
+                ", provenance='" + provenance + '\'' +
+                '}';
+    }
+}

src/main/java/eu/openaire/urls_worker/security/SecurityConfiguration.java

@@ -0,0 +1,50 @@
+package eu.openaire.urls_worker.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.config.http.SessionCreationPolicy;
+
+
+@Configuration
+@EnableWebSecurity
+@EnableGlobalMethodSecurity (
+        securedEnabled = false, // Just for now..
+        jsr250Enabled = true,
+        prePostEnabled = true
+)
+public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
+
+    private static final Logger logger = LoggerFactory.getLogger(SecurityConfiguration.class);
+
+
+    // Defines which resources are public and which are secured.
+    @Override
+    protected void configure(HttpSecurity http) throws Exception {
+        http
+                .headers()
+                .frameOptions()
+                .sameOrigin()
+                .and()
+                .cors()
+                .and()
+                .csrf()
+                .disable()
+                .exceptionHandling()
+                .and()
+                .sessionManagement()
+                .sessionCreationPolicy(SessionCreationPolicy.STATELESS)
+                .and()
+                .authorizeRequests()
+                .antMatchers("/**").permitAll()
+                //.anyRequest().authenticated()
+                //.and()
+                //.requiresChannel()
+                //.anyRequest().requiresSecure()
+        ;
+    }
+}

src/main/java/eu/openaire/urls_worker/util/UriBuilder.java

@@ -0,0 +1,61 @@
+package eu.openaire.urls_worker.util;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.env.Environment;
+
+import java.net.InetAddress;
+
+public class UriBuilder {
+
+    private static final Logger logger = LoggerFactory.getLogger(UriBuilder.class);
+
+    public static String baseUrl = null;
+
+    public UriBuilder(Environment environment) {
+        baseUrl = "http";
+
+        String sslEnabled = environment.getProperty("server.ssl.enabled");
+        if (sslEnabled == null) { // It's expected to not exist if there is no SSL-configuration.
+            logger.warn("No property \"server.ssl.enabled\" was found in \"application.properties\". Continuing with plain HTTP..");
+            sslEnabled = "false";
+        }
+        baseUrl += sslEnabled.equals("true") ? "s" : "";
+
+        baseUrl += "://";
+
+        String hostName = InetAddress.getLoopbackAddress().getHostName();   // Non-null.
+        baseUrl += hostName;
+
+        String serverPort = environment.getProperty("server.port");
+        if (serverPort == null) { // This is unacceptable!
+            logger.error("No property \"server.port\" was found in \"application.properties\"!");
+            System.exit(-1);    // Well, I guess the Spring Boot would not start in this case anyway.
+        }
+        baseUrl += ":" + serverPort;
+
+        String baseInternalPath = environment.getProperty("server.servlet.context-path");
+        if (baseInternalPath != null) {
+            if (!baseInternalPath.startsWith("/"))
+                baseUrl += "/";
+            baseUrl += baseInternalPath;
+            if (!baseInternalPath.endsWith("/"))
+                baseUrl += "/";
+        } else {
+            logger.warn("No property \"server.servlet.context-path\" was found in \"application.properties\"!");    // Yes it's expected.
+            baseUrl += "/";
+        }
+
+        logger.debug("ServerBaseURL: " + baseUrl);
+    }
+
+    public static String getBaseUrl() {
+        return baseUrl;
+    }
+
+    public static void setBaseUrl(String baseUrl) {
+        UriBuilder.baseUrl = baseUrl;
+    }
+
+}

src/main/resources/application.properties

@@ -0,0 +1,37 @@
+# HTTPS CONFIGURATION
+#server.port = 8444
+#server.ssl.enabled = true
+#server.ssl.key-store = src/main/resources/keystore.p12
+#server.ssl.key-store-type = PKCS12
+#server.ssl.key-alias = tomcat
+#server.ssl.key-store-password = urls_worker_project
+#server.tomcat.remoteip.remote-ip-header = x-your-remote-ip-header
+#server.tomcat.remoteip.protocol-header = x-your-protocol-header
+#server.error.include-stacktrace=never
+
+# HTTP CONFIGURATION
+server.port = 8081
+
+# Server api path
+server.servlet.context-path=/api
+
+# LOGGING LEVELS
+spring.output.ansi.enabled=always
+logging.level.root=WARN
+logging.level.org.springframework.web=INFO
+logging.level.eu.openaire.urls_worker=DEBUG
+
+
+## MULTIPART (MultipartProperties)
+
+# Enable multipart uploads
+spring.servlet.multipart.enabled=true
+
+# Threshold after which files are written to disk.
+spring.servlet.multipart.file-size-threshold=2KB
+
+# Max file size.
+spring.servlet.multipart.max-file-size=200MB
+
+# Max Request Size
+spring.servlet.multipart.max-request-size=215MB

src/main/resources/logback-spring.xml

@@ -0,0 +1,30 @@
+<configuration debug="false">
+
+    <appender name="File" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>logs/urls_worker.log</file>
+
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>logs/urls_worker.%i.log.zip</fileNamePattern>
+        </rollingPolicy>
+
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>50MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <charset>UTF-8</charset>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M\(@%line\) - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <charset>UTF-8</charset>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{36}.%M\(@%line\)) - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="info">
+        <appender-ref ref="Console" />
+    </root>
+
+</configuration>

src/test/java/eu/openaire/urls_worker/UrlsWorkerApplicationTests.java

@@ -0,0 +1,13 @@
+package eu.openaire.urls_worker;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest
+class UrlsWorkerApplicationTests {
+
+    @Test
+    void contextLoads() {
+    }
+
+}