diff --git a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java index 37cc55e..04a6d1c 100644 --- a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java +++ b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java @@ -6,6 +6,8 @@ import eu.openaire.urls_controller.models.Datasource; import eu.openaire.urls_controller.models.Task; import eu.openaire.urls_controller.payloads.requests.WorkerReport; import eu.openaire.urls_controller.payloads.responces.AssignmentResponse; +//import eu.openaire.urls_controller.repositories.AssignmentRepository; +import eu.openaire.urls_controller.util.ControllerConstants; import eu.openaire.urls_controller.util.FileUtils; import eu.openaire.urls_controller.util.GenericUtils; import org.slf4j.Logger; @@ -15,6 +17,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.*; +import java.util.concurrent.atomic.AtomicLong; @RestController @RequestMapping("/urls") @@ -23,26 +26,33 @@ public class UrlController { private static final Logger logger = LoggerFactory.getLogger(UrlController.class); - public UrlController() { + private static AtomicLong assignmentCounter = new AtomicLong(); // Just for the "getTestUrls"-endpoint. } @GetMapping("") - public ResponseEntity getUrls(@RequestParam String workerId, @RequestParam int tasksLimit) { + public ResponseEntity getUrls(@RequestParam String workerId, @RequestParam int workerTasksLimit) { List tasks = new ArrayList<>(); // TODO - Create the Assignment from the id-urls stored in the database up to the tasks-limit. - int assignmentId = 0; + Date date = new Date(); // Store it here, in order to have the same date for all current assignments. - Assignment assignment = new Assignment(assignmentId, tasks, workerId, new Date()); + + int tasksLimitForAssignment = ControllerConstants.ASSIGNMENTS_LIMIT; + if ( tasksLimitForAssignment > workerTasksLimit ) + tasksLimitForAssignment = workerTasksLimit; + + List assignments = null; // TODO -> // assignmentRepository.getNewAssignments(tasksLimitForAssignment); + + //Assignment assignment = new Assignment(assignmentId, tasks, workerId, date); // TODO - Write the Assignment details to the database and then send it to the worker. - logger.info("Sending assignment_" + assignment.getAssignmentId() + " to worker with ID: " + workerId); + logger.info("Sending assignment_" + assignmentCounter.get() + " to worker with ID: " + workerId); - return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment)); + return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments)); } @PostMapping("addWorkerReport") @@ -59,22 +69,21 @@ public class UrlController { } - private static int assignmentId = -1; // Just for the "getTestUrls"-endpoint. - @GetMapping("test") - public ResponseEntity getTestUrls(@RequestParam String workerId, @RequestParam int tasksLimit) { + public ResponseEntity getTestUrls(@RequestParam String workerId, @RequestParam int assignmentsLimit) { try { new FileUtils(); // Find the input file. } catch (Exception e) { logger.error(e.getMessage()); - return ResponseEntity.status(500).body("The resource file for the requested tasks was not found."); + return ResponseEntity.status(500).body("The resource file, for the requested assignments, was not found."); } - List tasks = new ArrayList<>(); + List assignments = new ArrayList<>(); HashMultimap loadedIdUrlPairs; boolean isFirstRun = true; - boolean tasksLimitReached = false; + boolean assignmentsLimitReached = false; + Date date = new Date(); // Start loading urls. while ( true ) { @@ -89,17 +98,17 @@ public class UrlController { for ( Map.Entry pair : pairs ) { - if ( tasks.size() >= tasksLimit ) { - tasksLimitReached = true; + if ( assignments.size() >= assignmentsLimit ) { + assignmentsLimitReached = true; break; } int randomNum = GenericUtils.getRandomNumber(1, 5); - tasks.add(new Task(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum))); + assignments.add(new Assignment(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum), workerId, date)); }// end pairs-for-loop - if ( tasksLimitReached ) { - logger.debug("Done loading urls from the inputFile as the tasksLimit (" + tasksLimit + ") was reached."); + if ( assignmentsLimitReached ) { + logger.debug("Done loading urls from the inputFile as the assignmentsLimit (" + assignmentsLimit + ") was reached."); break; } }// end loading-while-loop @@ -107,11 +116,9 @@ public class UrlController { if ( FileUtils.inputScanner.get() != null ) // Check if the initial value is null. FileUtils.inputScanner.get().close(); - Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date()); + logger.info("Sending AssignmentResponse_" + assignmentCounter.get() + " with " + assignments.size() + " assignments (" + FileUtils.duplicateIdUrlEntries.get() + " more assignments were discarded as duplicates), to worker with ID: " + workerId); - logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries.get() + " more tasks were discarded as duplicates), to worker with ID: " + workerId); - - return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment)); + return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments)); } } diff --git a/src/main/java/eu/openaire/urls_controller/models/Assignment.java b/src/main/java/eu/openaire/urls_controller/models/Assignment.java index 006779f..7da1fa5 100644 --- a/src/main/java/eu/openaire/urls_controller/models/Assignment.java +++ b/src/main/java/eu/openaire/urls_controller/models/Assignment.java @@ -4,52 +4,80 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +//import javax.persistence.*; +import java.io.Serializable; import java.util.Date; import java.util.List; +//@Entity +//@Table(name = "assignment") @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ - "assignmentId", - "tasks", + "id", + "original_url", + "datasource", "workerId", "date" }) -public class Assignment { +public class Assignment implements Serializable { - @JsonProperty("assignmentId") - private int assignmentId; + //@Id + //@Column(name = "id") + @JsonProperty("id") + private String id; - @JsonProperty("tasks") - private List tasks; + //@Id + //@Column(name = "original_url") + @JsonProperty("original_url") + private String originalUrl; - @JsonProperty("workerId") + @JsonProperty("datasource") + private Datasource datasource; + + //@Column(name = "workerid") + @JsonProperty("workerid") private String workerId; + //@Column(name = "date") @JsonProperty("date") private Date date; - public Assignment(int assignmentId, List tasks, String workerId, Date date) { - this.assignmentId = assignmentId; - this.tasks = tasks; + + public Assignment() {} + + + public Assignment(String id, String originalUrl, Datasource datasource, String workerId, Date date) { + this.id = id; + this.originalUrl = originalUrl; + this.datasource = datasource; this.workerId = workerId; this.date = date; } - public int getAssignmentId() { - return assignmentId; + + public String getId() { + return id; } - public void setAssignmentId(int assignmentId) { - this.assignmentId = assignmentId; + public void setId(String id) { + this.id = id; } - public List getTasks() { - return tasks; + public String getOriginalUrl() { + return originalUrl; } - public void setTasks(List tasks) { - this.tasks = tasks; + public void setOriginalUrl(String originalUrl) { + this.originalUrl = originalUrl; + } + + public Datasource getDatasource() { + return datasource; + } + + public void setDatasource(Datasource datasource) { + this.datasource = datasource; } public String getWorkerId() { @@ -71,8 +99,9 @@ public class Assignment { @Override public String toString() { return "Assignment{" + - "assignmentId=" + assignmentId + - ", tasks=" + tasks + + "id='" + id + '\'' + + ", originalUrl='" + originalUrl + '\'' + + ", datasource=" + datasource + ", workerId='" + workerId + '\'' + ", date=" + date + '}'; diff --git a/src/main/java/eu/openaire/urls_controller/models/Payload.java b/src/main/java/eu/openaire/urls_controller/models/Payload.java index 664b728..81462c0 100644 --- a/src/main/java/eu/openaire/urls_controller/models/Payload.java +++ b/src/main/java/eu/openaire/urls_controller/models/Payload.java @@ -4,64 +4,72 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +//import javax.persistence.Column; +import java.io.Serializable; import java.util.Date; +//@Entity +//@Table(name = "payload") @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ "id", "original_url", "actual_url", - "date_acquired", + "date", "mime_type", "size", - "more_info", - "md5", + "hash", "location", "provenance" }) -public class Payload { +public class Payload implements Serializable { + //@Column(name = "id") @JsonProperty("id") private String id; + //@Column(name = "original_url") @JsonProperty("original_url") private String original_url; + //@Column(name = "date") @JsonProperty("actual_url") private String actual_url; - @JsonProperty("date_acquired") + //@Column(name = "date") + @JsonProperty("date") private Date date_acquired; + //@Column(name = "mimetype") @JsonProperty("mime_type") private String mime_type; + //@Column(name = "size") @JsonProperty("size") private Long size; // In bytes. - @JsonProperty("more_info") - private String more_info; - - @JsonProperty("md5") - private String md5; + //@Column(name = "hash") + @JsonProperty("hash") + private String hash; + //@Column(name = "location") @JsonProperty("location") private String location; + //@Column(name = "provenance") @JsonProperty("provenance") - private String provenance; + private String provenance; // "crawl:" - public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String more_info, String md5, String location, String provenance) { + public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String hash, String location, String provenance) { this.id = id; this.original_url = original_url; this.actual_url = actual_url; this.date_acquired = date_acquired; this.mime_type = mime_type; this.size = size; - this.more_info = more_info; - this.md5 = md5; + this.hash = hash; this.location = location; this.provenance = provenance; } @@ -114,20 +122,12 @@ public class Payload { this.size = size; } - public String getMore_info() { - return more_info; + public String getHash() { + return hash; } - public void setMore_info(String more_info) { - this.more_info = more_info; - } - - public String getMd5() { - return md5; - } - - public void setMd5(String md5) { - this.md5 = md5; + public void setHash(String hash) { + this.hash = hash; } public String getLocation() { @@ -155,8 +155,7 @@ public class Payload { ", date_acquired='" + date_acquired + '\'' + ", mime_type='" + mime_type + '\'' + ", size='" + size + '\'' + - ", more_info='" + more_info + '\'' + - ", md5='" + md5 + '\'' + + ", md5='" + hash + '\'' + ", location='" + location + '\'' + ", provenance='" + provenance + '\'' + '}'; diff --git a/src/main/java/eu/openaire/urls_controller/payloads/requests/WorkerReport.java b/src/main/java/eu/openaire/urls_controller/payloads/requests/WorkerReport.java index 23f9a5f..013842c 100644 --- a/src/main/java/eu/openaire/urls_controller/payloads/requests/WorkerReport.java +++ b/src/main/java/eu/openaire/urls_controller/payloads/requests/WorkerReport.java @@ -19,15 +19,15 @@ public class WorkerReport { @JsonProperty("workerId") private String workerId; - @JsonProperty("assignmentId") - private int assignmentId; + @JsonProperty("assignmentRequestCounter") + private Long assignmentRequestCounter; @JsonProperty("urlReports") private List urlReports; - public WorkerReport(String workerId, int assignmentId, List urlReports) { + public WorkerReport(String workerId, Long assignmentRequestCounter, List urlReports) { this.workerId = workerId; - this.assignmentId = assignmentId; + this.assignmentRequestCounter = assignmentRequestCounter; this.urlReports = urlReports; } @@ -39,12 +39,12 @@ public class WorkerReport { this.workerId = workerId; } - public int getAssignmentId() { - return this.assignmentId; + public Long getAssignmentRequestCounter() { + return assignmentRequestCounter; } - public void setAssignmentId(int assignmentId) { - this.assignmentId = assignmentId; + public void setAssignmentRequestCounter(Long assignmentRequestCounter) { + this.assignmentRequestCounter = assignmentRequestCounter; } public List getUrlReports() { @@ -59,7 +59,7 @@ public class WorkerReport { public String toString() { return "WorkerReport{" + "workerId='" + workerId + '\'' + - ", assignmentId=" + assignmentId + + ", assignmentRequestCounter=" + assignmentRequestCounter + ", urlReports=" + urlReports + '}'; } diff --git a/src/main/java/eu/openaire/urls_controller/payloads/responces/AssignmentResponse.java b/src/main/java/eu/openaire/urls_controller/payloads/responces/AssignmentResponse.java index f8bdb83..b935a5a 100644 --- a/src/main/java/eu/openaire/urls_controller/payloads/responces/AssignmentResponse.java +++ b/src/main/java/eu/openaire/urls_controller/payloads/responces/AssignmentResponse.java @@ -4,29 +4,44 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import eu.openaire.urls_controller.models.Assignment; +import java.util.List; + @JsonInclude(JsonInclude.Include.NON_NULL) public class AssignmentResponse { - @JsonProperty("assignment") - private Assignment assignment; + @JsonProperty("assignmentCounter") + private Long assignmentCounter; - public AssignmentResponse(Assignment assignment) { - this.assignment = assignment; + @JsonProperty("assignments") + private List assignments; + + public AssignmentResponse(Long assignmentCounter, List assignments) { + this.assignmentCounter = assignmentCounter; + this.assignments = assignments; } - public Assignment getAssignment() { - return assignment; + public Long getAssignmentCounter() { + return assignmentCounter; } - public void setAssignment(Assignment assignment) { - this.assignment = assignment; + public void setAssignmentCounter(Long assignmentCounter) { + this.assignmentCounter = assignmentCounter; + } + + public List getAssignments() { + return assignments; + } + + public void setAssignments(List assignments) { + this.assignments = assignments; } @Override public String toString() { return "AssignmentResponse{" + - "assignment=" + assignment + + "assignments=" + assignments + '}'; } + } diff --git a/src/main/java/eu/openaire/urls_controller/util/ControllerConstants.java b/src/main/java/eu/openaire/urls_controller/util/ControllerConstants.java new file mode 100644 index 0000000..15faa27 --- /dev/null +++ b/src/main/java/eu/openaire/urls_controller/util/ControllerConstants.java @@ -0,0 +1,7 @@ +package eu.openaire.urls_controller.util; + +public interface ControllerConstants { + + int ASSIGNMENTS_LIMIT = 10000; // The general assignments-limit the Controller will get. If the worker cannot handle them, then the worker's limit will be applied. + +}