diff --git a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java index 47e0929..b17ce66 100644 --- a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java @@ -2,7 +2,7 @@ package eu.openaire.urls_worker.components; import eu.openaire.urls_worker.controllers.FullTextsController; import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin; -import eu.openaire.urls_worker.util.AssignmentHandler; +import eu.openaire.urls_worker.util.AssignmentsHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; @@ -32,8 +32,8 @@ public class ScheduledTasks { @Scheduled(fixedRate = 900_000) // Every 15 mins: 900_000 public void handleNewAssignments() { - if ( AssignmentHandler.isAvailableForWork ) - AssignmentHandler.handleAssignments(); + if ( AssignmentsHandler.isAvailableForWork ) + AssignmentsHandler.handleAssignments(); else { //logger.debug("The worker is not available for work at the moment.."); // JUST FOR DEBUG! } diff --git a/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java b/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java index 7afbbf4..4112c73 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java @@ -2,7 +2,7 @@ package eu.openaire.urls_worker.controllers; import eu.openaire.urls_worker.UrlsWorkerApplication; import eu.openaire.urls_worker.payloads.responces.WorkerResponse; -import eu.openaire.urls_worker.util.AssignmentHandler; +import eu.openaire.urls_worker.util.AssignmentsHandler; import eu.openaire.urls_worker.util.WorkerConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ public class GeneralController { logger.info("Received an \"isWorkerAvailableForWork\" request."); - if ( AssignmentHandler.isAvailableForWork ) { + if ( AssignmentsHandler.isAvailableForWork ) { logger.info("The worker is available for an assignment."); return ResponseEntity.status(200).body(new WorkerResponse(UrlsWorkerApplication.workerId, WorkerConstants.ASSIGNMENTS_LIMIT)); } else { diff --git a/src/main/java/eu/openaire/urls_worker/models/Assignment.java b/src/main/java/eu/openaire/urls_worker/models/Assignment.java index ef00dee..c38eccc 100644 --- a/src/main/java/eu/openaire/urls_worker/models/Assignment.java +++ b/src/main/java/eu/openaire/urls_worker/models/Assignment.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.util.Date; +import java.sql.Timestamp; @JsonInclude(JsonInclude.Include.NON_NULL) @@ -13,7 +13,7 @@ import java.util.Date; "original_url", "datasource", "workerId", - "date" + "timestamp" }) public class Assignment { @@ -29,19 +29,18 @@ public class Assignment { @JsonProperty("workerid") private String workerId; - @JsonProperty("date") - private Date date; + @JsonProperty("timestamp") + private Timestamp timestamp; public Assignment() {} - - public Assignment(String id, String originalUrl, Datasource datasource, String workerId, Date date) { + public Assignment(String id, String originalUrl, Datasource datasource, String workerId, Timestamp timestamp) { this.id = id; this.originalUrl = originalUrl; this.datasource = datasource; this.workerId = workerId; - this.date = date; + this.timestamp = timestamp; } @@ -77,15 +76,14 @@ public class Assignment { this.workerId = workerId; } - public Date getDate() { - return date; + public Timestamp getTimestamp() { + return timestamp; } - public void setDate(Date date) { - this.date = date; + public void setTimestamp(Timestamp timestamp) { + this.timestamp = timestamp; } - @Override public String toString() { return "Assignment{" + @@ -93,7 +91,7 @@ public class Assignment { ", originalUrl='" + originalUrl + '\'' + ", datasource=" + datasource + ", workerId='" + workerId + '\'' + - ", date=" + date + + ", timestamp=" + timestamp + '}'; } } diff --git a/src/main/java/eu/openaire/urls_worker/models/Payload.java b/src/main/java/eu/openaire/urls_worker/models/Payload.java index 55c3526..4b618f6 100644 --- a/src/main/java/eu/openaire/urls_worker/models/Payload.java +++ b/src/main/java/eu/openaire/urls_worker/models/Payload.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import java.util.Date; +import java.sql.Timestamp; @JsonInclude(JsonInclude.Include.NON_NULL) @@ -12,7 +12,7 @@ import java.util.Date; "id", "original_url", "actual_url", - "date", + "timestamp_acquired", "mime_type", "size", "hash", @@ -30,8 +30,8 @@ public class Payload { @JsonProperty("actual_url") private String actual_url; - @JsonProperty("date") - private Date date_acquired; + @JsonProperty("timestamp_acquired") + private Timestamp timestamp_acquired; @JsonProperty("mime_type") private String mime_type; @@ -48,12 +48,13 @@ public class Payload { @JsonProperty("provenance") private String provenance; // "crawl:" + public Payload() {} - public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String hash, String location, String provenance) { + public Payload(String id, String original_url, String actual_url, Timestamp timestamp_acquired, String mime_type, Long size, String hash, String location, String provenance) { this.id = id; this.original_url = original_url; this.actual_url = actual_url; - this.date_acquired = date_acquired; + this.timestamp_acquired = timestamp_acquired; this.mime_type = mime_type; this.size = size; this.hash = hash; @@ -85,12 +86,12 @@ public class Payload { this.actual_url = actual_url; } - public Date getDate_acquired() { - return date_acquired; + public Timestamp getTimestamp_acquired() { + return timestamp_acquired; } - public void setDate_acquired(Date date_acquired) { - this.date_acquired = date_acquired; + public void setTimestamp_acquired(Timestamp timestamp_acquired) { + this.timestamp_acquired = timestamp_acquired; } public String getMime_type() { @@ -139,7 +140,7 @@ public class Payload { "id='" + id + '\'' + ", original_url='" + original_url + '\'' + ", actual_url='" + actual_url + '\'' + - ", date_acquired='" + date_acquired + '\'' + + ", timestamp_acquired='" + timestamp_acquired + '\'' + ", mime_type='" + mime_type + '\'' + ", size='" + size + '\'' + ", hash='" + hash + '\'' + diff --git a/src/main/java/eu/openaire/urls_worker/payloads/requests/AssignmentRequest.java b/src/main/java/eu/openaire/urls_worker/payloads/requests/AssignmentsRequest.java similarity index 55% rename from src/main/java/eu/openaire/urls_worker/payloads/requests/AssignmentRequest.java rename to src/main/java/eu/openaire/urls_worker/payloads/requests/AssignmentsRequest.java index b2ec1f7..c6f4144 100644 --- a/src/main/java/eu/openaire/urls_worker/payloads/requests/AssignmentRequest.java +++ b/src/main/java/eu/openaire/urls_worker/payloads/requests/AssignmentsRequest.java @@ -8,27 +8,27 @@ import java.util.List; @JsonInclude(JsonInclude.Include.NON_NULL) -public class AssignmentRequest { +public class AssignmentsRequest { - @JsonProperty("assignmentCounter") - private Long assignmentCounter; + @JsonProperty("assignmentsCounter") + private Long assignmentsCounter; @JsonProperty("assignments") private List assignments; - public AssignmentRequest() { } + public AssignmentsRequest() { } - public AssignmentRequest(Long assignmentCounter, List assignments) { - this.assignmentCounter = assignmentCounter; + public AssignmentsRequest(Long assignmentCounter, List assignments) { + this.assignmentsCounter = assignmentCounter; this.assignments = assignments; } - public Long getAssignmentCounter() { - return assignmentCounter; + public Long getAssignmentsCounter() { + return assignmentsCounter; } - public void setAssignmentCounter(Long assignmentCounter) { - this.assignmentCounter = assignmentCounter; + public void setAssignmentsCounter(Long assignmentsCounter) { + this.assignmentsCounter = assignmentsCounter; } public List getAssignments() { @@ -41,8 +41,8 @@ public class AssignmentRequest { @Override public String toString() { - return "AssignmentRequest{" + - "assignmentCounter=" + assignmentCounter + + return "AssignmentsRequest{" + + "assignmentCounter=" + assignmentsCounter + ", assignments=" + assignments + '}'; } diff --git a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java index 6f81f1b..5473904 100644 --- a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java @@ -14,13 +14,14 @@ import eu.openaire.urls_worker.models.Error; import eu.openaire.urls_worker.models.Payload; import eu.openaire.urls_worker.models.UrlReport; import eu.openaire.urls_worker.services.FileStorageService; -import eu.openaire.urls_worker.util.AssignmentHandler; +import eu.openaire.urls_worker.util.AssignmentsHandler; import eu.openaire.urls_worker.util.WorkerConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.Executors; @@ -136,7 +137,7 @@ public class PublicationsRetrieverPlugin { public static void addUrlReportsToWorkerReport() { - Date date = new Date(); + Timestamp timestamp = new Timestamp(System.currentTimeMillis()); // Store it here, in order to have the same for all current records. for ( DataToBeLogged data : FileUtils.dataToBeLoggedList ) { @@ -191,10 +192,10 @@ public class PublicationsRetrieverPlugin { if ( (hash != null) && (hash.equals("null")) ) hash = null; - Payload payload = new Payload(data.getUrlId(), data.getSourceUrl(), docOrDatasetUrl, date, mimeType, size, hash, fileLocation, "crawl:PublicationsRetriever"); + Payload payload = new Payload(data.getUrlId(), data.getSourceUrl(), docOrDatasetUrl, timestamp, mimeType, size, hash, fileLocation, "crawl:PublicationsRetriever"); // TODO - If support is added for other doc-formats other than "pdf", then make sure the "mime_type" is correctly specified. - AssignmentHandler.urlReports.add(new UrlReport(status, payload, error)); + AssignmentsHandler.urlReports.add(new UrlReport(status, payload, error)); }// end-for FileUtils.dataToBeLoggedList.clear(); // Empty the list, to be re-populated by the next batch / assignment. } diff --git a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java b/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java similarity index 93% rename from src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java rename to src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java index 36021fb..380edd5 100644 --- a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java +++ b/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java @@ -5,7 +5,7 @@ import com.google.common.collect.Multimap; import eu.openaire.urls_worker.UrlsWorkerApplication; import eu.openaire.urls_worker.models.Assignment; import eu.openaire.urls_worker.models.UrlReport; -import eu.openaire.urls_worker.payloads.requests.AssignmentRequest; +import eu.openaire.urls_worker.payloads.requests.AssignmentsRequest; import eu.openaire.urls_worker.payloads.responces.WorkerReport; import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin; import org.slf4j.Logger; @@ -20,9 +20,9 @@ import java.time.Duration; import java.util.*; -public class AssignmentHandler { +public class AssignmentsHandler { - private static final Logger logger = LoggerFactory.getLogger(AssignmentHandler.class); + private static final Logger logger = LoggerFactory.getLogger(AssignmentsHandler.class); public static boolean isAvailableForWork = true; public static final List urlReports = new ArrayList<>(WorkerConstants.ASSIGNMENTS_LIMIT); @@ -37,14 +37,14 @@ public class AssignmentHandler { public static final RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(requestReadTimeoutDuration).build(); - public static AssignmentRequest requestAssignments() + public static AssignmentsRequest requestAssignments() { String requestUrl = UrlsWorkerApplication.controllerBaseUrl + "urls" + (askForTest ? "/test" : "") + "?workerId=" + UrlsWorkerApplication.workerId + "&workerAssignmentsLimit=" + WorkerConstants.ASSIGNMENTS_LIMIT; logger.info("Going to request assignments from the controller-server: " + requestUrl); - AssignmentRequest assignmentRequest = null; + AssignmentsRequest assignmentRequest = null; try { // Here, the HTTP-request is executed. - assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentRequest.class); + assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class); } catch (RestClientException rce) { logger.error("Could not retrieve the assignments!\n" + rce.getMessage()); // It shows the response body (after Spring v.2.5.6). return null; @@ -57,12 +57,12 @@ public class AssignmentHandler { public static void handleAssignments() { - AssignmentRequest assignmentRequest = requestAssignments(); - if ( assignmentRequest == null ) + AssignmentsRequest assignmentsRequest = requestAssignments(); + if ( assignmentsRequest == null ) return; - Long assignmentRequestCounter = assignmentRequest.getAssignmentCounter(); - List assignments = assignmentRequest.getAssignments(); + Long assignmentRequestCounter = assignmentsRequest.getAssignmentsCounter(); + List assignments = assignmentsRequest.getAssignments(); if ( assignments == null ) { logger.warn("The assignments were found to be null for assignmentRequestCounter = " + assignmentRequestCounter); return;