- Refactor the Assignment-creation.

In order to match the database, now we have a list of Assignments sent through the AssignmentResponse, instead of a single Assignment having a list of tasks.
- Cleanup the members of the "Payload" model (also prepare for database integration).

src/main/java/eu/openaire/urls_controller/controllers/UrlController.java

@@ -6,6 +6,8 @@ import eu.openaire.urls_controller.models.Datasource;
 import eu.openaire.urls_controller.models.Task;
 import eu.openaire.urls_controller.payloads.requests.WorkerReport;
 import eu.openaire.urls_controller.payloads.responces.AssignmentResponse;
+//import eu.openaire.urls_controller.repositories.AssignmentRepository;
+import eu.openaire.urls_controller.util.ControllerConstants;
 import eu.openaire.urls_controller.util.FileUtils;
 import eu.openaire.urls_controller.util.GenericUtils;
 import org.slf4j.Logger;
@@ -15,6 +17,7 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 
 import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
 
 @RestController
 @RequestMapping("/urls")
@@ -23,26 +26,33 @@ public class UrlController {
     private static final Logger logger = LoggerFactory.getLogger(UrlController.class);
 
 
-    public UrlController() {
+    private static AtomicLong assignmentCounter = new AtomicLong();    // Just for the "getTestUrls"-endpoint.
 
     }
 
 
     @GetMapping("")
-    public ResponseEntity<?> getUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
+    public ResponseEntity<?> getUrls(@RequestParam String workerId, @RequestParam int workerTasksLimit) {
 
         List<Task> tasks = new ArrayList<>();
 
         // TODO - Create the Assignment from the id-urls stored in the database up to the tasks-limit.
-        int assignmentId = 0;
+        Date date = new Date(); // Store it here, in order to have the same date for all current assignments.
 
-        Assignment assignment = new Assignment(assignmentId, tasks, workerId, new Date());
+
+        int tasksLimitForAssignment = ControllerConstants.ASSIGNMENTS_LIMIT;
+        if ( tasksLimitForAssignment > workerTasksLimit )
+            tasksLimitForAssignment = workerTasksLimit;
+
+        List<Assignment> assignments = null; // TODO ->  // assignmentRepository.getNewAssignments(tasksLimitForAssignment);
+
+        //Assignment assignment = new Assignment(assignmentId, tasks, workerId, date);
 
         // TODO - Write the Assignment details to the database and then send it to the worker.
 
-        logger.info("Sending assignment_" + assignment.getAssignmentId() + " to worker with ID: " + workerId);
+        logger.info("Sending assignment_" + assignmentCounter.get() + " to worker with ID: " + workerId);
 
-        return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
+        return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments));
     }
 
     @PostMapping("addWorkerReport")
@@ -59,22 +69,21 @@ public class UrlController {
     }
 
 
-    private static int assignmentId = -1;    // Just for the "getTestUrls"-endpoint.
-
     @GetMapping("test")
-    public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
+    public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int assignmentsLimit) {
 
         try {
             new FileUtils();    // Find the input file.
         } catch (Exception e) {
             logger.error(e.getMessage());
-            return ResponseEntity.status(500).body("The resource file for the requested tasks was not found.");
+            return ResponseEntity.status(500).body("The resource file, for the requested assignments, was not found.");
         }
 
-        List<Task> tasks = new ArrayList<>();
+        List<Assignment> assignments = new ArrayList<>();
         HashMultimap<String, String> loadedIdUrlPairs;
         boolean isFirstRun = true;
-        boolean tasksLimitReached = false;
+        boolean assignmentsLimitReached = false;
+        Date date = new Date();
 
         // Start loading urls.
         while ( true ) {
@@ -89,17 +98,17 @@ public class UrlController {
 
             for ( Map.Entry<String,String> pair : pairs )
             {
-                if ( tasks.size() >= tasksLimit ) {
+                if ( assignments.size() >= assignmentsLimit ) {
-                    tasksLimitReached = true;
+                    assignmentsLimitReached = true;
                     break;
                 }
 
                 int randomNum = GenericUtils.getRandomNumber(1, 5);
-                tasks.add(new Task(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum)));
+                assignments.add(new Assignment(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum), workerId, date));
             }// end pairs-for-loop
 
-            if ( tasksLimitReached ) {
+            if ( assignmentsLimitReached ) {
-                logger.debug("Done loading urls from the inputFile as the tasksLimit (" + tasksLimit + ") was reached.");
+                logger.debug("Done loading urls from the inputFile as the assignmentsLimit (" + assignmentsLimit + ") was reached.");
                 break;
             }
         }// end loading-while-loop
@@ -107,11 +116,9 @@ public class UrlController {
         if ( FileUtils.inputScanner.get() != null ) // Check if the initial value is null.
             FileUtils.inputScanner.get().close();
 
-        Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date());
+        logger.info("Sending AssignmentResponse_" +  assignmentCounter.get() + " with " + assignments.size() + " assignments (" + FileUtils.duplicateIdUrlEntries.get() + " more assignments were discarded as duplicates), to worker with ID: " + workerId);
 
-        logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries.get() + " more tasks were discarded as duplicates), to worker with ID: " + workerId);
+        return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments));
-
-        return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
     }
 
 }

src/main/java/eu/openaire/urls_controller/models/Assignment.java

@@ -4,52 +4,80 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 
+//import javax.persistence.*;
+import java.io.Serializable;
 import java.util.Date;
 import java.util.List;
 
 
+//@Entity
+//@Table(name = "assignment")
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonPropertyOrder({
-        "assignmentId",
+        "id",
-        "tasks",
+        "original_url",
+        "datasource",
         "workerId",
         "date"
 })
-public class Assignment {
+public class Assignment implements Serializable {
 
-    @JsonProperty("assignmentId")
+    //@Id
-    private int assignmentId;
+    //@Column(name = "id")
+    @JsonProperty("id")
+    private String id;
 
-    @JsonProperty("tasks")
+    //@Id
-    private List<Task> tasks;
+    //@Column(name = "original_url")
+    @JsonProperty("original_url")
+    private String originalUrl;
 
-    @JsonProperty("workerId")
+    @JsonProperty("datasource")
+    private Datasource datasource;
+
+    //@Column(name = "workerid")
+    @JsonProperty("workerid")
     private String workerId;
 
+    //@Column(name = "date")
     @JsonProperty("date")
     private Date date;
 
-    public Assignment(int assignmentId, List<Task> tasks, String workerId, Date date) {
+
-        this.assignmentId = assignmentId;
+    public Assignment() {}
-        this.tasks = tasks;
+
+
+    public Assignment(String id, String originalUrl, Datasource datasource, String workerId, Date date) {
+        this.id = id;
+        this.originalUrl = originalUrl;
+        this.datasource = datasource;
         this.workerId = workerId;
         this.date = date;
     }
 
-    public int getAssignmentId() {
+
-        return assignmentId;
+    public String getId() {
+        return id;
     }
 
-    public void setAssignmentId(int assignmentId) {
+    public void setId(String id) {
-        this.assignmentId = assignmentId;
+        this.id = id;
     }
 
-    public List<Task> getTasks() {
+    public String getOriginalUrl() {
-        return tasks;
+        return originalUrl;
     }
 
-    public void setTasks(List<Task> tasks) {
+    public void setOriginalUrl(String originalUrl) {
-        this.tasks = tasks;
+        this.originalUrl = originalUrl;
+    }
+
+    public Datasource getDatasource() {
+        return datasource;
+    }
+
+    public void setDatasource(Datasource datasource) {
+        this.datasource = datasource;
     }
 
     public String getWorkerId() {
@@ -71,8 +99,9 @@ public class Assignment {
     @Override
     public String toString() {
         return "Assignment{" +
-                "assignmentId=" + assignmentId +
+                "id='" + id + '\'' +
-                ", tasks=" + tasks +
+                ", originalUrl='" + originalUrl + '\'' +
+                ", datasource=" + datasource +
                 ", workerId='" + workerId + '\'' +
                 ", date=" + date +
                 '}';

src/main/java/eu/openaire/urls_controller/models/Payload.java

@@ -4,64 +4,72 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 
+//import javax.persistence.Column;
+import java.io.Serializable;
 import java.util.Date;
 
 
+//@Entity
+//@Table(name = "payload")
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonPropertyOrder({
         "id",
         "original_url",
         "actual_url",
-        "date_acquired",
+        "date",
         "mime_type",
         "size",
-        "more_info",
+        "hash",
-        "md5",
         "location",
         "provenance"
 })
-public class Payload {
+public class Payload implements Serializable {
 
+    //@Column(name = "id")
     @JsonProperty("id")
     private String id;
 
+    //@Column(name = "original_url")
     @JsonProperty("original_url")
     private String original_url;
 
+    //@Column(name = "date")
     @JsonProperty("actual_url")
     private String actual_url;
 
-    @JsonProperty("date_acquired")
+    //@Column(name = "date")
+    @JsonProperty("date")
     private Date date_acquired;
 
+    //@Column(name = "mimetype")
     @JsonProperty("mime_type")
     private String mime_type;
 
+    //@Column(name = "size")
     @JsonProperty("size")
     private Long size;  // In bytes.
 
-    @JsonProperty("more_info")
+    //@Column(name = "hash")
-    private String more_info;
+    @JsonProperty("hash")
-
+    private String hash;
-    @JsonProperty("md5")
-    private String md5;
 
+    //@Column(name = "location")
     @JsonProperty("location")
     private String location;
 
+    //@Column(name = "provenance")
     @JsonProperty("provenance")
-    private String provenance;
+    private String provenance;  // "crawl:<PluginName>"
 
 
-    public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String more_info, String md5, String location, String provenance) {
+    public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String hash, String location, String provenance) {
         this.id = id;
         this.original_url = original_url;
         this.actual_url = actual_url;
         this.date_acquired = date_acquired;
         this.mime_type = mime_type;
         this.size = size;
-        this.more_info = more_info;
+        this.hash = hash;
-        this.md5 = md5;
         this.location = location;
         this.provenance = provenance;
     }
@@ -114,20 +122,12 @@ public class Payload {
         this.size = size;
     }
 
-    public String getMore_info() {
+    public String getHash() {
-        return more_info;
+        return hash;
     }
 
-    public void setMore_info(String more_info) {
+    public void setHash(String hash) {
-        this.more_info = more_info;
+        this.hash = hash;
-    }
-
-    public String getMd5() {
-        return md5;
-    }
-
-    public void setMd5(String md5) {
-        this.md5 = md5;
     }
 
     public String getLocation() {
@@ -155,8 +155,7 @@ public class Payload {
                 ", date_acquired='" + date_acquired + '\'' +
                 ", mime_type='" + mime_type + '\'' +
                 ", size='" + size + '\'' +
-                ", more_info='" + more_info + '\'' +
+                ", md5='" + hash + '\'' +
-                ", md5='" + md5 + '\'' +
                 ", location='" + location + '\'' +
                 ", provenance='" + provenance + '\'' +
                 '}';

src/main/java/eu/openaire/urls_controller/payloads/requests/WorkerReport.java

@@ -19,15 +19,15 @@ public class WorkerReport {
     @JsonProperty("workerId")
     private String workerId;
 
-    @JsonProperty("assignmentId")
+    @JsonProperty("assignmentRequestCounter")
-    private int assignmentId;
+    private Long assignmentRequestCounter;
 
     @JsonProperty("urlReports")
     private List<UrlReport> urlReports;
 
-    public WorkerReport(String workerId, int assignmentId, List<UrlReport> urlReports) {
+    public WorkerReport(String workerId, Long assignmentRequestCounter, List<UrlReport> urlReports) {
         this.workerId = workerId;
-        this.assignmentId = assignmentId;
+        this.assignmentRequestCounter = assignmentRequestCounter;
         this.urlReports = urlReports;
     }
 
@@ -39,12 +39,12 @@ public class WorkerReport {
         this.workerId = workerId;
     }
 
-    public int getAssignmentId() {
+    public Long getAssignmentRequestCounter() {
-        return this.assignmentId;
+        return assignmentRequestCounter;
     }
 
-    public void setAssignmentId(int assignmentId) {
+    public void setAssignmentRequestCounter(Long assignmentRequestCounter) {
-        this.assignmentId = assignmentId;
+        this.assignmentRequestCounter = assignmentRequestCounter;
     }
 
     public List<UrlReport> getUrlReports() {
@@ -59,7 +59,7 @@ public class WorkerReport {
     public String toString() {
         return "WorkerReport{" +
                 "workerId='" + workerId + '\'' +
-                ", assignmentId=" + assignmentId +
+                ", assignmentRequestCounter=" + assignmentRequestCounter +
                 ", urlReports=" + urlReports +
                 '}';
     }

src/main/java/eu/openaire/urls_controller/payloads/responces/AssignmentResponse.java

@@ -4,29 +4,44 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import eu.openaire.urls_controller.models.Assignment;
 
+import java.util.List;
+
 
 @JsonInclude(JsonInclude.Include.NON_NULL)
 public class AssignmentResponse {
 
-    @JsonProperty("assignment")
+    @JsonProperty("assignmentCounter")
-    private Assignment assignment;
+    private Long assignmentCounter;
 
-    public AssignmentResponse(Assignment assignment) {
+    @JsonProperty("assignments")
-        this.assignment = assignment;
+    private List<Assignment> assignments;
+
+    public AssignmentResponse(Long assignmentCounter, List<Assignment> assignments) {
+        this.assignmentCounter = assignmentCounter;
+        this.assignments = assignments;
     }
 
-    public Assignment getAssignment() {
+    public Long getAssignmentCounter() {
-        return assignment;
+        return assignmentCounter;
     }
 
-    public void setAssignment(Assignment assignment) {
+    public void setAssignmentCounter(Long assignmentCounter) {
-        this.assignment = assignment;
+        this.assignmentCounter = assignmentCounter;
+    }
+
+    public List<Assignment> getAssignments() {
+        return assignments;
+    }
+
+    public void setAssignments(List<Assignment> assignments) {
+        this.assignments = assignments;
     }
 
     @Override
     public String toString() {
         return "AssignmentResponse{" +
-                "assignment=" + assignment +
+                "assignments=" + assignments +
                 '}';
     }
+
 }

src/main/java/eu/openaire/urls_controller/util/ControllerConstants.java

@@ -0,0 +1,7 @@
+package eu.openaire.urls_controller.util;
+
+public interface ControllerConstants {
+
+    int ASSIGNMENTS_LIMIT = 10000;    // The general assignments-limit the Controller will get. If the worker cannot handle them, then the worker's limit will be applied.
+
+}