- Refactor the Assignment-creation.

In order to match the database, now we have a list of Assignments sent through the AssignmentResponse, instead of a single Assignment having a list of tasks.
- Cleanup the members of the "Payload" model (also prepare for database integration).
This commit is contained in:
Lampros Smyrnaios 2021-07-05 14:04:39 +03:00
parent 5e7ccbd8c6
commit 27375b9396
6 changed files with 145 additions and 88 deletions

View File

@ -6,6 +6,8 @@ import eu.openaire.urls_controller.models.Datasource;
import eu.openaire.urls_controller.models.Task;
import eu.openaire.urls_controller.payloads.requests.WorkerReport;
import eu.openaire.urls_controller.payloads.responces.AssignmentResponse;
//import eu.openaire.urls_controller.repositories.AssignmentRepository;
import eu.openaire.urls_controller.util.ControllerConstants;
import eu.openaire.urls_controller.util.FileUtils;
import eu.openaire.urls_controller.util.GenericUtils;
import org.slf4j.Logger;
@ -15,6 +17,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
@RestController
@RequestMapping("/urls")
@ -23,26 +26,33 @@ public class UrlController {
private static final Logger logger = LoggerFactory.getLogger(UrlController.class);
public UrlController() {
private static AtomicLong assignmentCounter = new AtomicLong(); // Just for the "getTestUrls"-endpoint.
}
@GetMapping("")
public ResponseEntity<?> getUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
public ResponseEntity<?> getUrls(@RequestParam String workerId, @RequestParam int workerTasksLimit) {
List<Task> tasks = new ArrayList<>();
// TODO - Create the Assignment from the id-urls stored in the database up to the tasks-limit.
int assignmentId = 0;
Date date = new Date(); // Store it here, in order to have the same date for all current assignments.
Assignment assignment = new Assignment(assignmentId, tasks, workerId, new Date());
int tasksLimitForAssignment = ControllerConstants.ASSIGNMENTS_LIMIT;
if ( tasksLimitForAssignment > workerTasksLimit )
tasksLimitForAssignment = workerTasksLimit;
List<Assignment> assignments = null; // TODO -> // assignmentRepository.getNewAssignments(tasksLimitForAssignment);
//Assignment assignment = new Assignment(assignmentId, tasks, workerId, date);
// TODO - Write the Assignment details to the database and then send it to the worker.
logger.info("Sending assignment_" + assignment.getAssignmentId() + " to worker with ID: " + workerId);
logger.info("Sending assignment_" + assignmentCounter.get() + " to worker with ID: " + workerId);
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments));
}
@PostMapping("addWorkerReport")
@ -59,22 +69,21 @@ public class UrlController {
}
private static int assignmentId = -1; // Just for the "getTestUrls"-endpoint.
@GetMapping("test")
public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int assignmentsLimit) {
try {
new FileUtils(); // Find the input file.
} catch (Exception e) {
logger.error(e.getMessage());
return ResponseEntity.status(500).body("The resource file for the requested tasks was not found.");
return ResponseEntity.status(500).body("The resource file, for the requested assignments, was not found.");
}
List<Task> tasks = new ArrayList<>();
List<Assignment> assignments = new ArrayList<>();
HashMultimap<String, String> loadedIdUrlPairs;
boolean isFirstRun = true;
boolean tasksLimitReached = false;
boolean assignmentsLimitReached = false;
Date date = new Date();
// Start loading urls.
while ( true ) {
@ -89,17 +98,17 @@ public class UrlController {
for ( Map.Entry<String,String> pair : pairs )
{
if ( tasks.size() >= tasksLimit ) {
tasksLimitReached = true;
if ( assignments.size() >= assignmentsLimit ) {
assignmentsLimitReached = true;
break;
}
int randomNum = GenericUtils.getRandomNumber(1, 5);
tasks.add(new Task(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum)));
assignments.add(new Assignment(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum), workerId, date));
}// end pairs-for-loop
if ( tasksLimitReached ) {
logger.debug("Done loading urls from the inputFile as the tasksLimit (" + tasksLimit + ") was reached.");
if ( assignmentsLimitReached ) {
logger.debug("Done loading urls from the inputFile as the assignmentsLimit (" + assignmentsLimit + ") was reached.");
break;
}
}// end loading-while-loop
@ -107,11 +116,9 @@ public class UrlController {
if ( FileUtils.inputScanner.get() != null ) // Check if the initial value is null.
FileUtils.inputScanner.get().close();
Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date());
logger.info("Sending AssignmentResponse_" + assignmentCounter.get() + " with " + assignments.size() + " assignments (" + FileUtils.duplicateIdUrlEntries.get() + " more assignments were discarded as duplicates), to worker with ID: " + workerId);
logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries.get() + " more tasks were discarded as duplicates), to worker with ID: " + workerId);
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments));
}
}

View File

@ -4,52 +4,80 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
//import javax.persistence.*;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
//@Entity
//@Table(name = "assignment")
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({
"assignmentId",
"tasks",
"id",
"original_url",
"datasource",
"workerId",
"date"
})
public class Assignment {
public class Assignment implements Serializable {
@JsonProperty("assignmentId")
private int assignmentId;
//@Id
//@Column(name = "id")
@JsonProperty("id")
private String id;
@JsonProperty("tasks")
private List<Task> tasks;
//@Id
//@Column(name = "original_url")
@JsonProperty("original_url")
private String originalUrl;
@JsonProperty("workerId")
@JsonProperty("datasource")
private Datasource datasource;
//@Column(name = "workerid")
@JsonProperty("workerid")
private String workerId;
//@Column(name = "date")
@JsonProperty("date")
private Date date;
public Assignment(int assignmentId, List<Task> tasks, String workerId, Date date) {
this.assignmentId = assignmentId;
this.tasks = tasks;
public Assignment() {}
public Assignment(String id, String originalUrl, Datasource datasource, String workerId, Date date) {
this.id = id;
this.originalUrl = originalUrl;
this.datasource = datasource;
this.workerId = workerId;
this.date = date;
}
public int getAssignmentId() {
return assignmentId;
public String getId() {
return id;
}
public void setAssignmentId(int assignmentId) {
this.assignmentId = assignmentId;
public void setId(String id) {
this.id = id;
}
public List<Task> getTasks() {
return tasks;
public String getOriginalUrl() {
return originalUrl;
}
public void setTasks(List<Task> tasks) {
this.tasks = tasks;
public void setOriginalUrl(String originalUrl) {
this.originalUrl = originalUrl;
}
public Datasource getDatasource() {
return datasource;
}
public void setDatasource(Datasource datasource) {
this.datasource = datasource;
}
public String getWorkerId() {
@ -71,8 +99,9 @@ public class Assignment {
@Override
public String toString() {
return "Assignment{" +
"assignmentId=" + assignmentId +
", tasks=" + tasks +
"id='" + id + '\'' +
", originalUrl='" + originalUrl + '\'' +
", datasource=" + datasource +
", workerId='" + workerId + '\'' +
", date=" + date +
'}';

View File

@ -4,64 +4,72 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
//import javax.persistence.Column;
import java.io.Serializable;
import java.util.Date;
//@Entity
//@Table(name = "payload")
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({
"id",
"original_url",
"actual_url",
"date_acquired",
"date",
"mime_type",
"size",
"more_info",
"md5",
"hash",
"location",
"provenance"
})
public class Payload {
public class Payload implements Serializable {
//@Column(name = "id")
@JsonProperty("id")
private String id;
//@Column(name = "original_url")
@JsonProperty("original_url")
private String original_url;
//@Column(name = "date")
@JsonProperty("actual_url")
private String actual_url;
@JsonProperty("date_acquired")
//@Column(name = "date")
@JsonProperty("date")
private Date date_acquired;
//@Column(name = "mimetype")
@JsonProperty("mime_type")
private String mime_type;
//@Column(name = "size")
@JsonProperty("size")
private Long size; // In bytes.
@JsonProperty("more_info")
private String more_info;
@JsonProperty("md5")
private String md5;
//@Column(name = "hash")
@JsonProperty("hash")
private String hash;
//@Column(name = "location")
@JsonProperty("location")
private String location;
//@Column(name = "provenance")
@JsonProperty("provenance")
private String provenance;
private String provenance; // "crawl:<PluginName>"
public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String more_info, String md5, String location, String provenance) {
public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String hash, String location, String provenance) {
this.id = id;
this.original_url = original_url;
this.actual_url = actual_url;
this.date_acquired = date_acquired;
this.mime_type = mime_type;
this.size = size;
this.more_info = more_info;
this.md5 = md5;
this.hash = hash;
this.location = location;
this.provenance = provenance;
}
@ -114,20 +122,12 @@ public class Payload {
this.size = size;
}
public String getMore_info() {
return more_info;
public String getHash() {
return hash;
}
public void setMore_info(String more_info) {
this.more_info = more_info;
}
public String getMd5() {
return md5;
}
public void setMd5(String md5) {
this.md5 = md5;
public void setHash(String hash) {
this.hash = hash;
}
public String getLocation() {
@ -155,8 +155,7 @@ public class Payload {
", date_acquired='" + date_acquired + '\'' +
", mime_type='" + mime_type + '\'' +
", size='" + size + '\'' +
", more_info='" + more_info + '\'' +
", md5='" + md5 + '\'' +
", md5='" + hash + '\'' +
", location='" + location + '\'' +
", provenance='" + provenance + '\'' +
'}';

View File

@ -19,15 +19,15 @@ public class WorkerReport {
@JsonProperty("workerId")
private String workerId;
@JsonProperty("assignmentId")
private int assignmentId;
@JsonProperty("assignmentRequestCounter")
private Long assignmentRequestCounter;
@JsonProperty("urlReports")
private List<UrlReport> urlReports;
public WorkerReport(String workerId, int assignmentId, List<UrlReport> urlReports) {
public WorkerReport(String workerId, Long assignmentRequestCounter, List<UrlReport> urlReports) {
this.workerId = workerId;
this.assignmentId = assignmentId;
this.assignmentRequestCounter = assignmentRequestCounter;
this.urlReports = urlReports;
}
@ -39,12 +39,12 @@ public class WorkerReport {
this.workerId = workerId;
}
public int getAssignmentId() {
return this.assignmentId;
public Long getAssignmentRequestCounter() {
return assignmentRequestCounter;
}
public void setAssignmentId(int assignmentId) {
this.assignmentId = assignmentId;
public void setAssignmentRequestCounter(Long assignmentRequestCounter) {
this.assignmentRequestCounter = assignmentRequestCounter;
}
public List<UrlReport> getUrlReports() {
@ -59,7 +59,7 @@ public class WorkerReport {
public String toString() {
return "WorkerReport{" +
"workerId='" + workerId + '\'' +
", assignmentId=" + assignmentId +
", assignmentRequestCounter=" + assignmentRequestCounter +
", urlReports=" + urlReports +
'}';
}

View File

@ -4,29 +4,44 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import eu.openaire.urls_controller.models.Assignment;
import java.util.List;
@JsonInclude(JsonInclude.Include.NON_NULL)
public class AssignmentResponse {
@JsonProperty("assignment")
private Assignment assignment;
@JsonProperty("assignmentCounter")
private Long assignmentCounter;
public AssignmentResponse(Assignment assignment) {
this.assignment = assignment;
@JsonProperty("assignments")
private List<Assignment> assignments;
public AssignmentResponse(Long assignmentCounter, List<Assignment> assignments) {
this.assignmentCounter = assignmentCounter;
this.assignments = assignments;
}
public Assignment getAssignment() {
return assignment;
public Long getAssignmentCounter() {
return assignmentCounter;
}
public void setAssignment(Assignment assignment) {
this.assignment = assignment;
public void setAssignmentCounter(Long assignmentCounter) {
this.assignmentCounter = assignmentCounter;
}
public List<Assignment> getAssignments() {
return assignments;
}
public void setAssignments(List<Assignment> assignments) {
this.assignments = assignments;
}
@Override
public String toString() {
return "AssignmentResponse{" +
"assignment=" + assignment +
"assignments=" + assignments +
'}';
}
}

View File

@ -0,0 +1,7 @@
package eu.openaire.urls_controller.util;
public interface ControllerConstants {
int ASSIGNMENTS_LIMIT = 10000; // The general assignments-limit the Controller will get. If the worker cannot handle them, then the worker's limit will be applied.
}