- Add deletion, of the cookies in the newly-supported CookieManager, after each batch.
- Update the Spring-Security-code to use the "SecurityFilterChain", as the previous code was deprecated. - Update dependencies. - Code cleanup.
This commit is contained in:
parent
26cbb83b51
commit
d91732bc16
|
@ -1,5 +1,5 @@
|
||||||
plugins {
|
plugins {
|
||||||
id 'org.springframework.boot' version '2.7.0'
|
id 'org.springframework.boot' version '2.7.1'
|
||||||
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
|
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
|
||||||
id 'java'
|
id 'java'
|
||||||
}
|
}
|
||||||
|
@ -24,12 +24,13 @@ dependencies {
|
||||||
implementation("org.springframework.security:spring-security-core")
|
implementation("org.springframework.security:spring-security-core")
|
||||||
implementation("org.springframework.security:spring-security-web")
|
implementation("org.springframework.security:spring-security-web")
|
||||||
implementation("org.springframework.security:spring-security-config")
|
implementation("org.springframework.security:spring-security-config")
|
||||||
//implementation("io.jsonwebtoken:jjwt:0.9.1") // Use this in case we use auth-tokens later on.
|
|
||||||
|
|
||||||
implementation 'org.projectlombok:lombok:1.18.24'
|
implementation 'org.projectlombok:lombok:1.18.24'
|
||||||
|
|
||||||
|
//implementation group: 'io.jsonwebtoken', name: 'jjwt-api', version: '0.11.5' // Use this in case we use auth-tokens later on.
|
||||||
|
|
||||||
// Enable the validation annotations.
|
// Enable the validation annotations.
|
||||||
//implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
|
//implementation group: 'jakarta.validation', name: 'jakarta.validation-api', version: '3.0.2'
|
||||||
|
|
||||||
implementation ("eu.openaire:publications_retriever:1.0-SNAPSHOT") {
|
implementation ("eu.openaire:publications_retriever:1.0-SNAPSHOT") {
|
||||||
exclude group: 'ch.qos.logback', module: 'logback-core'
|
exclude group: 'ch.qos.logback', module: 'logback-core'
|
||||||
|
|
|
@ -11,7 +11,6 @@ import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.text.SimpleDateFormat;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
|
@ -21,8 +20,6 @@ public class ScheduledTasks {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
|
private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
|
||||||
|
|
||||||
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
|
|
||||||
|
|
||||||
|
|
||||||
@Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes.
|
@Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes.
|
||||||
public void handleNewAssignments() {
|
public void handleNewAssignments() {
|
||||||
|
|
|
@ -1,13 +1,13 @@
|
||||||
package eu.openaire.urls_worker.security;
|
package eu.openaire.urls_worker.security;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
import org.springframework.context.annotation.Bean;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
|
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
|
||||||
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
||||||
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
|
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
|
||||||
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
|
|
||||||
import org.springframework.security.config.http.SessionCreationPolicy;
|
import org.springframework.security.config.http.SessionCreationPolicy;
|
||||||
|
import org.springframework.security.web.SecurityFilterChain;
|
||||||
|
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
|
@ -17,14 +17,10 @@ import org.springframework.security.config.http.SessionCreationPolicy;
|
||||||
jsr250Enabled = true,
|
jsr250Enabled = true,
|
||||||
prePostEnabled = true
|
prePostEnabled = true
|
||||||
)
|
)
|
||||||
public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
|
public class SecurityConfiguration {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(SecurityConfiguration.class);
|
@Bean
|
||||||
|
public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
|
||||||
|
|
||||||
// Defines which resources are public and which are secured.
|
|
||||||
@Override
|
|
||||||
protected void configure(HttpSecurity http) throws Exception {
|
|
||||||
http
|
http
|
||||||
.headers()
|
.headers()
|
||||||
.frameOptions()
|
.frameOptions()
|
||||||
|
@ -46,5 +42,7 @@ public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
|
||||||
//.requiresChannel()
|
//.requiresChannel()
|
||||||
//.anyRequest().requiresSecure()
|
//.anyRequest().requiresSecure()
|
||||||
;
|
;
|
||||||
|
return http.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -2,6 +2,7 @@ package eu.openaire.urls_worker.util;
|
||||||
|
|
||||||
import com.google.common.collect.HashMultimap;
|
import com.google.common.collect.HashMultimap;
|
||||||
import com.google.common.collect.Multimap;
|
import com.google.common.collect.Multimap;
|
||||||
|
import eu.openaire.publications_retriever.util.http.HttpConnUtils;
|
||||||
import eu.openaire.publications_retriever.util.url.GenericUtils;
|
import eu.openaire.publications_retriever.util.url.GenericUtils;
|
||||||
import eu.openaire.publications_retriever.util.url.UrlUtils;
|
import eu.openaire.publications_retriever.util.url.UrlUtils;
|
||||||
import eu.openaire.urls_worker.UrlsWorkerApplication;
|
import eu.openaire.urls_worker.UrlsWorkerApplication;
|
||||||
|
@ -19,6 +20,7 @@ import org.springframework.http.ResponseEntity;
|
||||||
import org.springframework.web.client.RestClientException;
|
import org.springframework.web.client.RestClientException;
|
||||||
import org.springframework.web.client.RestTemplate;
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
import java.net.CookieStore;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -48,12 +50,15 @@ public class AssignmentsHandler {
|
||||||
public static final long idUrlsToHandleBeforeClearingDuplicateUrlsData = 1_000_000;
|
public static final long idUrlsToHandleBeforeClearingDuplicateUrlsData = 1_000_000;
|
||||||
|
|
||||||
|
|
||||||
|
private static CookieStore cookieStore = null;
|
||||||
|
|
||||||
public AssignmentsHandler()
|
public AssignmentsHandler()
|
||||||
{
|
{
|
||||||
urlReports = new ArrayList<>(UrlsWorkerApplication.maxAssignmentsLimitPerBatch);
|
urlReports = new ArrayList<>(UrlsWorkerApplication.maxAssignmentsLimitPerBatch);
|
||||||
int expectedAssignmentsPerDatasource = (UrlsWorkerApplication.maxAssignmentsLimitPerBatch / expectedDatasourcesPerRequest);
|
int expectedAssignmentsPerDatasource = (UrlsWorkerApplication.maxAssignmentsLimitPerBatch / expectedDatasourcesPerRequest);
|
||||||
assignmentsForPlugins = HashMultimap.create(expectedDatasourcesPerRequest, expectedAssignmentsPerDatasource);
|
assignmentsForPlugins = HashMultimap.create(expectedDatasourcesPerRequest, expectedAssignmentsPerDatasource);
|
||||||
requestUrl = UrlsWorkerApplication.controllerBaseUrl + (askForTest ? "test/" : "") + "urls?workerId=" + UrlsWorkerApplication.workerId + "&workerAssignmentsLimit=" + UrlsWorkerApplication.maxAssignmentsLimitPerBatch;
|
requestUrl = UrlsWorkerApplication.controllerBaseUrl + (askForTest ? "test/" : "") + "urls?workerId=" + UrlsWorkerApplication.workerId + "&workerAssignmentsLimit=" + UrlsWorkerApplication.maxAssignmentsLimitPerBatch;
|
||||||
|
cookieStore = HttpConnUtils.cookieManager.getCookieStore();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -180,6 +185,9 @@ public class AssignmentsHandler {
|
||||||
} finally {
|
} finally {
|
||||||
urlReports.clear(); // Reset, without de-allocating.
|
urlReports.clear(); // Reset, without de-allocating.
|
||||||
assignmentsForPlugins.clear();
|
assignmentsForPlugins.clear();
|
||||||
|
logger.debug("The number of cookies is: " + cookieStore.getCookies().size());
|
||||||
|
boolean cookiesDeleted = cookieStore.removeAll();
|
||||||
|
logger.debug(cookiesDeleted ? "The cookies where removed!" : "No cookies where removed!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,10 @@ package eu.openaire.urls_worker.util;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
Loading…
Reference in New Issue