|
|
|
@ -2,6 +2,7 @@ package eu.openaire.urls_worker.util;
|
|
|
|
|
|
|
|
|
|
import com.google.common.collect.HashMultimap;
|
|
|
|
|
import com.google.common.collect.Multimap;
|
|
|
|
|
import eu.openaire.publications_retriever.util.http.HttpConnUtils;
|
|
|
|
|
import eu.openaire.publications_retriever.util.url.GenericUtils;
|
|
|
|
|
import eu.openaire.publications_retriever.util.url.UrlUtils;
|
|
|
|
|
import eu.openaire.urls_worker.UrlsWorkerApplication;
|
|
|
|
@ -19,6 +20,7 @@ import org.springframework.http.ResponseEntity;
|
|
|
|
|
import org.springframework.web.client.RestClientException;
|
|
|
|
|
import org.springframework.web.client.RestTemplate;
|
|
|
|
|
|
|
|
|
|
import java.net.CookieStore;
|
|
|
|
|
import java.time.Duration;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.List;
|
|
|
|
@ -48,12 +50,15 @@ public class AssignmentsHandler {
|
|
|
|
|
public static final long idUrlsToHandleBeforeClearingDuplicateUrlsData = 1_000_000;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static CookieStore cookieStore = null;
|
|
|
|
|
|
|
|
|
|
public AssignmentsHandler()
|
|
|
|
|
{
|
|
|
|
|
urlReports = new ArrayList<>(UrlsWorkerApplication.maxAssignmentsLimitPerBatch);
|
|
|
|
|
int expectedAssignmentsPerDatasource = (UrlsWorkerApplication.maxAssignmentsLimitPerBatch / expectedDatasourcesPerRequest);
|
|
|
|
|
assignmentsForPlugins = HashMultimap.create(expectedDatasourcesPerRequest, expectedAssignmentsPerDatasource);
|
|
|
|
|
requestUrl = UrlsWorkerApplication.controllerBaseUrl + (askForTest ? "test/" : "") + "urls?workerId=" + UrlsWorkerApplication.workerId + "&workerAssignmentsLimit=" + UrlsWorkerApplication.maxAssignmentsLimitPerBatch;
|
|
|
|
|
cookieStore = HttpConnUtils.cookieManager.getCookieStore();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -180,6 +185,9 @@ public class AssignmentsHandler {
|
|
|
|
|
} finally {
|
|
|
|
|
urlReports.clear(); // Reset, without de-allocating.
|
|
|
|
|
assignmentsForPlugins.clear();
|
|
|
|
|
logger.debug("The number of cookies is: " + cookieStore.getCookies().size());
|
|
|
|
|
boolean cookiesDeleted = cookieStore.removeAll();
|
|
|
|
|
logger.debug(cookiesDeleted ? "The cookies where removed!" : "No cookies where removed!");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|