- Add deletion, of the cookies in the newly-supported CookieManager, after each batch.

- Update the Spring-Security-code to use the "SecurityFilterChain", as the previous code was deprecated.
- Update dependencies.
- Code cleanup.
master
Lampros Smyrnaios 2 years ago
parent 26cbb83b51
commit d91732bc16

@ -1,5 +1,5 @@
plugins {
id 'org.springframework.boot' version '2.7.0'
id 'org.springframework.boot' version '2.7.1'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java'
}
@ -24,12 +24,13 @@ dependencies {
implementation("org.springframework.security:spring-security-core")
implementation("org.springframework.security:spring-security-web")
implementation("org.springframework.security:spring-security-config")
//implementation("io.jsonwebtoken:jjwt:0.9.1") // Use this in case we use auth-tokens later on.
implementation 'org.projectlombok:lombok:1.18.24'
//implementation group: 'io.jsonwebtoken', name: 'jjwt-api', version: '0.11.5' // Use this in case we use auth-tokens later on.
// Enable the validation annotations.
//implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
//implementation group: 'jakarta.validation', name: 'jakarta.validation-api', version: '3.0.2'
implementation ("eu.openaire:publications_retriever:1.0-SNAPSHOT") {
exclude group: 'ch.qos.logback', module: 'logback-core'

@ -11,7 +11,6 @@ import org.springframework.stereotype.Component;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Map;
import java.util.Set;
@ -21,8 +20,6 @@ public class ScheduledTasks {
private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
@Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes.
public void handleNewAssignments() {

@ -1,13 +1,13 @@
package eu.openaire.urls_worker.security;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.web.SecurityFilterChain;
@Configuration
@ -17,14 +17,10 @@ import org.springframework.security.config.http.SessionCreationPolicy;
jsr250Enabled = true,
prePostEnabled = true
)
public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
private static final Logger logger = LoggerFactory.getLogger(SecurityConfiguration.class);
public class SecurityConfiguration {
// Defines which resources are public and which are secured.
@Override
protected void configure(HttpSecurity http) throws Exception {
@Bean
public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
http
.headers()
.frameOptions()
@ -46,5 +42,7 @@ public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
//.requiresChannel()
//.anyRequest().requiresSecure()
;
return http.build();
}
}
}

@ -2,6 +2,7 @@ package eu.openaire.urls_worker.util;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import eu.openaire.publications_retriever.util.http.HttpConnUtils;
import eu.openaire.publications_retriever.util.url.GenericUtils;
import eu.openaire.publications_retriever.util.url.UrlUtils;
import eu.openaire.urls_worker.UrlsWorkerApplication;
@ -19,6 +20,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import java.net.CookieStore;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
@ -48,12 +50,15 @@ public class AssignmentsHandler {
public static final long idUrlsToHandleBeforeClearingDuplicateUrlsData = 1_000_000;
private static CookieStore cookieStore = null;
public AssignmentsHandler()
{
urlReports = new ArrayList<>(UrlsWorkerApplication.maxAssignmentsLimitPerBatch);
int expectedAssignmentsPerDatasource = (UrlsWorkerApplication.maxAssignmentsLimitPerBatch / expectedDatasourcesPerRequest);
assignmentsForPlugins = HashMultimap.create(expectedDatasourcesPerRequest, expectedAssignmentsPerDatasource);
requestUrl = UrlsWorkerApplication.controllerBaseUrl + (askForTest ? "test/" : "") + "urls?workerId=" + UrlsWorkerApplication.workerId + "&workerAssignmentsLimit=" + UrlsWorkerApplication.maxAssignmentsLimitPerBatch;
cookieStore = HttpConnUtils.cookieManager.getCookieStore();
}
@ -180,6 +185,9 @@ public class AssignmentsHandler {
} finally {
urlReports.clear(); // Reset, without de-allocating.
assignmentsForPlugins.clear();
logger.debug("The number of cookies is: " + cookieStore.getCookies().size());
boolean cookiesDeleted = cookieStore.removeAll();
logger.debug(cookiesDeleted ? "The cookies where removed!" : "No cookies where removed!");
}
}

@ -3,7 +3,10 @@ package eu.openaire.urls_worker.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.List;

Loading…
Cancel
Save