Initial commit of UrlsController.

README.md

@@ -0,0 +1,2 @@
+# UrlsController
+

build.gradle

@@ -0,0 +1,46 @@
+buildscript {
+    ext {
+        springSecurityVersion = "5.4.5"
+    }
+}
+
+plugins {
+    id 'org.springframework.boot' version '2.4.3'
+    id 'java'
+}
+
+apply plugin: 'java'
+apply plugin: 'io.spring.dependency-management'
+
+group = 'eu.openaire.urls_controller'
+version = '0.0.1-SNAPSHOT'
+sourceCompatibility = '1.8'
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    runtimeOnly 'org.springframework.boot:spring-boot-devtools'
+
+    implementation 'org.springframework.boot:spring-boot-starter-web'
+    implementation("org.springframework.boot:spring-boot-starter-security")
+    implementation("org.springframework.boot:spring-boot-configuration-processor")
+    implementation("org.springframework.security:spring-security-core:${springSecurityVersion}")
+    implementation("org.springframework.security:spring-security-web:${springSecurityVersion}")
+    implementation("org.springframework.security:spring-security-config:${springSecurityVersion}")
+    implementation("io.jsonwebtoken:jjwt:0.9.1")
+
+
+    implementation "org.projectlombok:lombok:1.18.18"
+    implementation 'com.google.code.gson:gson:2.8.6'
+    implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
+    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
+    implementation group: 'commons-io', name: 'commons-io', version: '2.8.0'
+
+    // https://mvnrepository.com/artifact/com.google.guava/guava
+    // implementation group: 'com.google.guava', name: 'guava', version: '30.1-jre' // It will be usefull later..
+
+    testImplementation group: 'org.springframework.security', name: 'spring-security-test', version: springSecurityVersion
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
+}

scripts/startServer.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -x
+cd ../
+./gradlew bootrun

settings.gradle

@@ -0,0 +1,6 @@
+pluginManagement {
+    repositories {
+        gradlePluginPortal()
+    }
+}
+rootProject.name = 'urls_controller'

src/main/java/eu/openaire/urls_controller/Application.java

@@ -0,0 +1,45 @@
+package eu.openaire.urls_controller;
+
+
+import eu.openaire.urls_controller.util.UriBuilder;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.core.env.Environment;
+import org.springframework.web.cors.CorsConfiguration;
+import org.springframework.web.cors.CorsConfigurationSource;
+import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+@SpringBootApplication
+@EnableScheduling
+public class Application {
+
+    public static void main(String[] args) {
+        SpringApplication.run(Application.class, args);
+    }
+
+    @Bean
+    public CorsConfigurationSource corsConfigurationSource() {
+        CorsConfiguration configuration = new CorsConfiguration();
+        configuration.setAllowedOrigins(Collections.singletonList("*"));
+        configuration.setAllowedMethods(Arrays.asList("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"));
+        configuration.setAllowedHeaders(Arrays.asList("authorization", "content-type", "x-auth-token"));
+        configuration.setExposedHeaders(Collections.singletonList("x-auth-token"));
+        UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
+        source.registerCorsConfiguration("/**", configuration);
+        return source;
+    }
+
+
+    @Bean
+    public CommandLineRunner setServerBaseUrl(Environment environment)
+    {
+        return args -> new UriBuilder(environment);
+    }
+
+}

src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java

@@ -0,0 +1,23 @@
+package eu.openaire.urls_controller.components;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+
+@Component
+public class ScheduledTasks {
+
+    private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
+
+    private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
+    @Scheduled(fixedRate = 600_000)   // TODO - Change to every 10 mins: 600_000
+    public void reportCurrentTime() {
+        logger.info("Server is live! Time is now {}", dateFormat.format(new Date()));
+    }
+}

src/main/java/eu/openaire/urls_controller/controllers/UrlController.java

@@ -0,0 +1,71 @@
+package eu.openaire.urls_controller.controllers;
+
+import eu.openaire.urls_controller.models.UrlToCheck;
+import eu.openaire.urls_controller.payloads.responces.UrlsResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@RestController
+@RequestMapping("/urls")
+public class UrlController {
+
+    private static final Logger logger = LoggerFactory.getLogger(UrlController.class);
+
+
+    public UrlController() {
+
+    }
+
+
+    @GetMapping("")
+    public ResponseEntity<?> getUrls() {
+
+        List<UrlToCheck> urlsToCheck = new ArrayList<>();
+
+
+        // TODO - Retrieve the ID-URL pairs from the database.
+
+        // For now, we give static data.
+        urlsToCheck.add(new UrlToCheck("50|dedup_wf_001::173a4e29249d4a67e72148ff50a88968", "http://dx.doi.org/10.1590/s0034-76122012000200012"));
+        urlsToCheck.add(new UrlToCheck("50|dedup_wf_001::220cb83d8a6e7a90ac4ac72feb030700", "https://doaj.org/toc/2214-9147"));
+        urlsToCheck.add(new UrlToCheck("50|dedup_wf_001::2b29092d3ed25d7ccfac894974e0915e", "http://www.plantintroduction.org/index.php/pi/article/view/1201"));
+        urlsToCheck.add(new UrlToCheck("50|dedup_wf_001::2fed1ea2149efd0d5a7837240a4aaf71", "http://europepmc.org/articles/pmc2882784?pdf=render"));
+        urlsToCheck.add(new UrlToCheck("50|dedup_wf_001::180e60bbb541a9b9a9313779887da9cf", "http://dx.doi.org/10.1051/e3sconf/202016405016"));
+
+        logger.debug("ID-URL pairs to return:\n" + urlsToCheck);
+
+        return ResponseEntity.status(200).header("Content-Type", "application/json").body(new UrlsResponse(urlsToCheck).toString());
+    }
+
+
+
+    @PostMapping("")
+    // TODO - Add authorization to edit the database.
+    public ResponseEntity<?> addResults() {
+
+        UrlToCheck urlToCheck = new UrlToCheck();
+
+        List<UrlToCheck> urlsToCheck = new ArrayList<>();
+
+
+        // TODO - Retrieve the ID-URL pairs from the database.
+
+        // For now, we give static data.
+        urlsToCheck.add(new UrlToCheck("ID-1", "URL-1"));
+
+
+        logger.debug("ID-URL pairs to return: " + urlsToCheck);
+
+        return ResponseEntity.ok(new UrlsResponse(urlsToCheck));
+    }
+
+
+}

src/main/java/eu/openaire/urls_controller/exceptions/BadRequestException.java

@@ -0,0 +1,16 @@
+package eu.openaire.urls_controller.exceptions;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseStatus(HttpStatus.BAD_REQUEST)
+public class BadRequestException extends RuntimeException {
+
+    public BadRequestException(String message) {
+        super(message);
+    }
+
+    public BadRequestException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/eu/openaire/urls_controller/models/Datasource.java

@@ -0,0 +1,29 @@
+package eu.openaire.urls_controller.models;
+
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({
+        "id",
+        "name"
+})
+public class Datasource {
+
+    @JsonProperty("id")
+    String id;
+
+    @JsonProperty("name")
+    String name;
+
+
+    @Override
+    public String toString() {
+        return "Datasource{" +
+                "id='" + id + '\'' +
+                ", name='" + name + '\'' +
+                '}';
+    }
+}

src/main/java/eu/openaire/urls_controller/models/FullText.java

@@ -0,0 +1,45 @@
+package eu.openaire.urls_controller.models;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({
+        "id",
+        "link_to_payload"
+})
+public class FullText {
+
+    @JsonProperty("id")
+    private String id;
+
+    @JsonProperty("link_to_payload")
+    private String link_to_payload;
+
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getLink_to_payload() {
+        return link_to_payload;
+    }
+
+    public void setLink_to_payload(String link_to_payload) {
+        this.link_to_payload = link_to_payload;
+    }
+
+    @Override
+    public String toString() {
+        return "FullText{" +
+                "id='" + id + '\'' +
+                ", link_to_payload='" + link_to_payload + '\'' +
+                '}';
+    }
+}

src/main/java/eu/openaire/urls_controller/models/Payload.java

@@ -0,0 +1,150 @@
+package eu.openaire.urls_controller.models;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({
+        "id",
+        "original_url",
+        "actual_url",
+        "date_acquired",
+        "mime_type",
+        "size",
+        "more_info",
+        "md5",
+        "location",
+        "provenance"
+})
+public class Payload {
+
+    @JsonProperty("id")
+    private String id;
+
+    @JsonProperty("original_url")
+    private String original_url;
+
+    @JsonProperty("actual_url")
+    private String actual_url;
+
+    @JsonProperty("date_acquired")
+    private String date_acquired;
+
+    @JsonProperty("mime_type")
+    private String mime_type;
+
+    @JsonProperty("size")
+    private String size;
+
+    @JsonProperty("more_info")
+    private  String more_info;
+
+    @JsonProperty("md5")
+    private String md5;
+
+    @JsonProperty("location")
+    private String location;
+
+    @JsonProperty("provenance")
+    private String provenance;
+
+
+    public String getOriginal_url() {
+        return original_url;
+    }
+
+    public void setOriginal_url(String original_url) {
+        this.original_url = original_url;
+    }
+
+    public String getActual_url() {
+        return actual_url;
+    }
+
+    public void setActual_url(String actual_url) {
+        this.actual_url = actual_url;
+    }
+
+    public String getDate_acquired() {
+        return date_acquired;
+    }
+
+    public void setDate_acquired(String date_acquired) {
+        this.date_acquired = date_acquired;
+    }
+
+    public String getMime_type() {
+        return mime_type;
+    }
+
+    public void setMime_type(String mime_type) {
+        this.mime_type = mime_type;
+    }
+
+    public String getSize() {
+        return size;
+    }
+
+    public void setSize(String size) {
+        this.size = size;
+    }
+
+    public String getMore_info() {
+        return more_info;
+    }
+
+    public void setMore_info(String more_info) {
+        this.more_info = more_info;
+    }
+
+    public String getMd5() {
+        return md5;
+    }
+
+    public void setMd5(String md5) {
+        this.md5 = md5;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    public void setLocation(String location) {
+        this.location = location;
+    }
+
+    public String getProvenance() {
+        return provenance;
+    }
+
+    public void setProvenance(String provenance) {
+        this.provenance = provenance;
+    }
+
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    @Override
+    public String toString() {
+        return "Payload{" +
+                "id='" + id + '\'' +
+                ", original_url='" + original_url + '\'' +
+                ", actual_url='" + actual_url + '\'' +
+                ", date_acquired='" + date_acquired + '\'' +
+                ", mime_type='" + mime_type + '\'' +
+                ", size='" + size + '\'' +
+                ", more_info='" + more_info + '\'' +
+                ", md5='" + md5 + '\'' +
+                ", location='" + location + '\'' +
+                ", provenance='" + provenance + '\'' +
+                '}';
+    }
+}

src/main/java/eu/openaire/urls_controller/models/Publication.java

@@ -0,0 +1,29 @@
+package eu.openaire.urls_controller.models;
+
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({
+        "id",
+        "dedupid",
+        "terms_of_reuse"
+})
+public class Publication {
+
+
+    @JsonProperty("id")
+    private String id;
+
+
+    @JsonProperty("dedupid")
+    private String dedupid;
+
+    @JsonProperty("terms_of_reuse")
+    private String terms_of_reuse;
+
+}
+

src/main/java/eu/openaire/urls_controller/models/UrlToCheck.java

@@ -0,0 +1,53 @@
+package eu.openaire.urls_controller.models;
+
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({
+        "id",
+        "url"
+})
+public class UrlToCheck {  // This model will not match with a database,
+
+
+    @JsonProperty("id")
+    private String id;
+
+
+    @JsonProperty("url")
+    private String url;
+
+
+    public UrlToCheck() {
+    }
+
+    public UrlToCheck(String id, String url) {
+        this.id = id;
+        this.url = url;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    @Override
+    public String toString() {
+        return "{\"id\":\"" + id + "\",\"url\":\"" + url + "\"}\n";
+    }
+}

src/main/java/eu/openaire/urls_controller/payloads/responces/UrlsResponse.java

@@ -0,0 +1,38 @@
+package eu.openaire.urls_controller.payloads.responces;
+
+import eu.openaire.urls_controller.models.UrlToCheck;
+
+import java.util.List;
+
+public class UrlsResponse {
+
+    private List<UrlToCheck> urlsToCheck;
+
+    public UrlsResponse() {
+    }
+
+    public UrlsResponse(List<UrlToCheck> urlsToCheck) {
+        this.urlsToCheck = urlsToCheck;
+    }
+
+    public UrlsResponse get() {
+        return this;
+    }
+
+    public List<UrlToCheck> getUrlsToCheck() {
+        return urlsToCheck;
+    }
+
+    public void setUrlsToCheck(List<UrlToCheck> urlToCheck) {
+        this.urlsToCheck = urlToCheck;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder(urlsToCheck.size());
+        for ( UrlToCheck urlToCheck : urlsToCheck ) {
+            sb.append(urlToCheck);
+        }
+        return sb.toString();
+    }
+}

src/main/java/eu/openaire/urls_controller/security/SecurityConfiguration.java

@@ -0,0 +1,53 @@
+package eu.openaire.urls_controller.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.authentication.AuthenticationManager;
+import org.springframework.security.config.BeanIds;
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.config.http.SessionCreationPolicy;
+
+
+@Configuration
+@EnableWebSecurity
+@EnableGlobalMethodSecurity (
+        securedEnabled = false, // Just for now..
+        jsr250Enabled = true,
+        prePostEnabled = true
+)
+public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
+
+    private static final Logger logger = LoggerFactory.getLogger(SecurityConfiguration.class);
+
+
+    // Defines which resources are public and which are secured.
+    @Override
+    protected void configure(HttpSecurity http) throws Exception {
+        http
+                .headers()
+                .frameOptions()
+                .sameOrigin()
+                .and()
+                .cors()
+                .and()
+                .csrf()
+                .disable()
+                .exceptionHandling()
+                .and()
+                .sessionManagement()
+                .sessionCreationPolicy(SessionCreationPolicy.STATELESS)
+                .and()
+                .authorizeRequests()
+                .antMatchers("/**").permitAll()
+                //.anyRequest().authenticated()
+                //.and()
+                //.requiresChannel()
+                //.anyRequest().requiresSecure()
+        ;
+    }
+}

src/main/java/eu/openaire/urls_controller/util/UriBuilder.java

@@ -0,0 +1,61 @@
+package eu.openaire.urls_controller.util;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.env.Environment;
+
+import java.net.InetAddress;
+
+public class UriBuilder {
+
+    private static final Logger logger = LoggerFactory.getLogger(UriBuilder.class);
+
+    public static String baseUrl = null;
+
+    public UriBuilder(Environment environment) {
+        baseUrl = "http";
+
+        String sslEnabled = environment.getProperty("server.ssl.enabled");
+        if (sslEnabled == null) { // It's expected to not exist if there is no SSL-configuration.
+            logger.warn("No property \"server.ssl.enabled\" was found in \"application.properties\". Continuing with plain HTTP..");
+            sslEnabled = "false";
+        }
+        baseUrl += sslEnabled.equals("true") ? "s" : "";
+
+        baseUrl += "://";
+
+        String hostName = InetAddress.getLoopbackAddress().getHostName();   // Non-null.
+        baseUrl += hostName;
+
+        String serverPort = environment.getProperty("server.port");
+        if (serverPort == null) { // This is unacceptable!
+            logger.error("No property \"server.port\" was found in \"application.properties\"!");
+            System.exit(-1);    // Well, I guess the Spring Boot would not start in this case anyway.
+        }
+        baseUrl += ":" + serverPort;
+
+        String baseInternalPath = environment.getProperty("server.servlet.context-path");
+        if (baseInternalPath != null) {
+            if (!baseInternalPath.startsWith("/"))
+                baseUrl += "/";
+            baseUrl += baseInternalPath;
+            if (!baseInternalPath.endsWith("/"))
+                baseUrl += "/";
+        } else {
+            logger.warn("No property \"server.servlet.context-path\" was found in \"application.properties\"!");    // Yes it's expected.
+            baseUrl += "/";
+        }
+
+        logger.debug("ServerBaseURL: " + baseUrl);
+    }
+
+    public static String getBaseUrl() {
+        return baseUrl;
+    }
+
+    public static void setBaseUrl(String baseUrl) {
+        UriBuilder.baseUrl = baseUrl;
+    }
+
+}

src/main/resources/application.properties

@@ -0,0 +1,37 @@
+# HTTPS CONFIGURATION
+#server.port = 8443
+#server.ssl.enabled = true
+#server.ssl.key-store = src/main/resources/keystore.p12
+#server.ssl.key-store-type = PKCS12
+#server.ssl.key-alias = tomcat
+#server.ssl.key-store-password = rent_project
+#server.tomcat.remoteip.remote-ip-header = x-your-remote-ip-header
+#server.tomcat.remoteip.protocol-header = x-your-protocol-header
+#server.error.include-stacktrace=never
+
+# HTTP CONFIGURATION
+server.port = 8080
+
+# Server api path
+server.servlet.context-path=/api
+
+# LOGGING LEVELS
+spring.output.ansi.enabled=always
+logging.level.root=WARN
+logging.level.org.springframework.web=INFO
+logging.level.eu.openaire.urls_controller=DEBUG
+
+
+## MULTIPART (MultipartProperties)
+
+# Enable multipart uploads
+spring.servlet.multipart.enabled=true
+
+# Threshold after which files are written to disk.
+spring.servlet.multipart.file-size-threshold=2KB
+
+# Max file size.
+spring.servlet.multipart.max-file-size=200MB
+
+# Max Request Size
+spring.servlet.multipart.max-request-size=215MB

src/main/resources/logback-spring.xml

@@ -0,0 +1,30 @@
+<configuration debug="false">
+
+    <appender name="File" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>logs/urls_controller.log</file>
+
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>logs/urls_controller.%i.log.zip</fileNamePattern>
+        </rollingPolicy>
+
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>50MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <charset>UTF-8</charset>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M\(@%line\) - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <charset>UTF-8</charset>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{36}.%M\(@%line\)) - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="info">
+        <appender-ref ref="Console" />
+    </root>
+
+</configuration>