forked from lsmyrnaios/UrlsController
- Refactor the Assignment-creation.
In order to match the database, now we have a list of Assignments sent through the AssignmentResponse, instead of a single Assignment having a list of tasks. - Cleanup the members of the "Payload" model (also prepare for database integration).
This commit is contained in:
parent
5e7ccbd8c6
commit
27375b9396
|
|
@ -6,6 +6,8 @@ import eu.openaire.urls_controller.models.Datasource;
|
|||
import eu.openaire.urls_controller.models.Task;
|
||||
import eu.openaire.urls_controller.payloads.requests.WorkerReport;
|
||||
import eu.openaire.urls_controller.payloads.responces.AssignmentResponse;
|
||||
//import eu.openaire.urls_controller.repositories.AssignmentRepository;
|
||||
import eu.openaire.urls_controller.util.ControllerConstants;
|
||||
import eu.openaire.urls_controller.util.FileUtils;
|
||||
import eu.openaire.urls_controller.util.GenericUtils;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -15,6 +17,7 @@ import org.springframework.http.ResponseEntity;
|
|||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/urls")
|
||||
|
|
@ -23,26 +26,33 @@ public class UrlController {
|
|||
private static final Logger logger = LoggerFactory.getLogger(UrlController.class);
|
||||
|
||||
|
||||
public UrlController() {
|
||||
private static AtomicLong assignmentCounter = new AtomicLong(); // Just for the "getTestUrls"-endpoint.
|
||||
|
||||
}
|
||||
|
||||
|
||||
@GetMapping("")
|
||||
public ResponseEntity<?> getUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
|
||||
public ResponseEntity<?> getUrls(@RequestParam String workerId, @RequestParam int workerTasksLimit) {
|
||||
|
||||
List<Task> tasks = new ArrayList<>();
|
||||
|
||||
// TODO - Create the Assignment from the id-urls stored in the database up to the tasks-limit.
|
||||
int assignmentId = 0;
|
||||
Date date = new Date(); // Store it here, in order to have the same date for all current assignments.
|
||||
|
||||
Assignment assignment = new Assignment(assignmentId, tasks, workerId, new Date());
|
||||
|
||||
int tasksLimitForAssignment = ControllerConstants.ASSIGNMENTS_LIMIT;
|
||||
if ( tasksLimitForAssignment > workerTasksLimit )
|
||||
tasksLimitForAssignment = workerTasksLimit;
|
||||
|
||||
List<Assignment> assignments = null; // TODO -> // assignmentRepository.getNewAssignments(tasksLimitForAssignment);
|
||||
|
||||
//Assignment assignment = new Assignment(assignmentId, tasks, workerId, date);
|
||||
|
||||
// TODO - Write the Assignment details to the database and then send it to the worker.
|
||||
|
||||
logger.info("Sending assignment_" + assignment.getAssignmentId() + " to worker with ID: " + workerId);
|
||||
logger.info("Sending assignment_" + assignmentCounter.get() + " to worker with ID: " + workerId);
|
||||
|
||||
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
|
||||
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments));
|
||||
}
|
||||
|
||||
@PostMapping("addWorkerReport")
|
||||
|
|
@ -59,22 +69,21 @@ public class UrlController {
|
|||
}
|
||||
|
||||
|
||||
private static int assignmentId = -1; // Just for the "getTestUrls"-endpoint.
|
||||
|
||||
@GetMapping("test")
|
||||
public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
|
||||
public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int assignmentsLimit) {
|
||||
|
||||
try {
|
||||
new FileUtils(); // Find the input file.
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage());
|
||||
return ResponseEntity.status(500).body("The resource file for the requested tasks was not found.");
|
||||
return ResponseEntity.status(500).body("The resource file, for the requested assignments, was not found.");
|
||||
}
|
||||
|
||||
List<Task> tasks = new ArrayList<>();
|
||||
List<Assignment> assignments = new ArrayList<>();
|
||||
HashMultimap<String, String> loadedIdUrlPairs;
|
||||
boolean isFirstRun = true;
|
||||
boolean tasksLimitReached = false;
|
||||
boolean assignmentsLimitReached = false;
|
||||
Date date = new Date();
|
||||
|
||||
// Start loading urls.
|
||||
while ( true ) {
|
||||
|
|
@ -89,17 +98,17 @@ public class UrlController {
|
|||
|
||||
for ( Map.Entry<String,String> pair : pairs )
|
||||
{
|
||||
if ( tasks.size() >= tasksLimit ) {
|
||||
tasksLimitReached = true;
|
||||
if ( assignments.size() >= assignmentsLimit ) {
|
||||
assignmentsLimitReached = true;
|
||||
break;
|
||||
}
|
||||
|
||||
int randomNum = GenericUtils.getRandomNumber(1, 5);
|
||||
tasks.add(new Task(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum)));
|
||||
assignments.add(new Assignment(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum), workerId, date));
|
||||
}// end pairs-for-loop
|
||||
|
||||
if ( tasksLimitReached ) {
|
||||
logger.debug("Done loading urls from the inputFile as the tasksLimit (" + tasksLimit + ") was reached.");
|
||||
if ( assignmentsLimitReached ) {
|
||||
logger.debug("Done loading urls from the inputFile as the assignmentsLimit (" + assignmentsLimit + ") was reached.");
|
||||
break;
|
||||
}
|
||||
}// end loading-while-loop
|
||||
|
|
@ -107,11 +116,9 @@ public class UrlController {
|
|||
if ( FileUtils.inputScanner.get() != null ) // Check if the initial value is null.
|
||||
FileUtils.inputScanner.get().close();
|
||||
|
||||
Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date());
|
||||
logger.info("Sending AssignmentResponse_" + assignmentCounter.get() + " with " + assignments.size() + " assignments (" + FileUtils.duplicateIdUrlEntries.get() + " more assignments were discarded as duplicates), to worker with ID: " + workerId);
|
||||
|
||||
logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries.get() + " more tasks were discarded as duplicates), to worker with ID: " + workerId);
|
||||
|
||||
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
|
||||
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignmentCounter.incrementAndGet(), assignments));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,52 +4,80 @@ import com.fasterxml.jackson.annotation.JsonInclude;
|
|||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
|
||||
//import javax.persistence.*;
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
//@Entity
|
||||
//@Table(name = "assignment")
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
@JsonPropertyOrder({
|
||||
"assignmentId",
|
||||
"tasks",
|
||||
"id",
|
||||
"original_url",
|
||||
"datasource",
|
||||
"workerId",
|
||||
"date"
|
||||
})
|
||||
public class Assignment {
|
||||
public class Assignment implements Serializable {
|
||||
|
||||
@JsonProperty("assignmentId")
|
||||
private int assignmentId;
|
||||
//@Id
|
||||
//@Column(name = "id")
|
||||
@JsonProperty("id")
|
||||
private String id;
|
||||
|
||||
@JsonProperty("tasks")
|
||||
private List<Task> tasks;
|
||||
//@Id
|
||||
//@Column(name = "original_url")
|
||||
@JsonProperty("original_url")
|
||||
private String originalUrl;
|
||||
|
||||
@JsonProperty("workerId")
|
||||
@JsonProperty("datasource")
|
||||
private Datasource datasource;
|
||||
|
||||
//@Column(name = "workerid")
|
||||
@JsonProperty("workerid")
|
||||
private String workerId;
|
||||
|
||||
//@Column(name = "date")
|
||||
@JsonProperty("date")
|
||||
private Date date;
|
||||
|
||||
public Assignment(int assignmentId, List<Task> tasks, String workerId, Date date) {
|
||||
this.assignmentId = assignmentId;
|
||||
this.tasks = tasks;
|
||||
|
||||
public Assignment() {}
|
||||
|
||||
|
||||
public Assignment(String id, String originalUrl, Datasource datasource, String workerId, Date date) {
|
||||
this.id = id;
|
||||
this.originalUrl = originalUrl;
|
||||
this.datasource = datasource;
|
||||
this.workerId = workerId;
|
||||
this.date = date;
|
||||
}
|
||||
|
||||
public int getAssignmentId() {
|
||||
return assignmentId;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setAssignmentId(int assignmentId) {
|
||||
this.assignmentId = assignmentId;
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public List<Task> getTasks() {
|
||||
return tasks;
|
||||
public String getOriginalUrl() {
|
||||
return originalUrl;
|
||||
}
|
||||
|
||||
public void setTasks(List<Task> tasks) {
|
||||
this.tasks = tasks;
|
||||
public void setOriginalUrl(String originalUrl) {
|
||||
this.originalUrl = originalUrl;
|
||||
}
|
||||
|
||||
public Datasource getDatasource() {
|
||||
return datasource;
|
||||
}
|
||||
|
||||
public void setDatasource(Datasource datasource) {
|
||||
this.datasource = datasource;
|
||||
}
|
||||
|
||||
public String getWorkerId() {
|
||||
|
|
@ -71,8 +99,9 @@ public class Assignment {
|
|||
@Override
|
||||
public String toString() {
|
||||
return "Assignment{" +
|
||||
"assignmentId=" + assignmentId +
|
||||
", tasks=" + tasks +
|
||||
"id='" + id + '\'' +
|
||||
", originalUrl='" + originalUrl + '\'' +
|
||||
", datasource=" + datasource +
|
||||
", workerId='" + workerId + '\'' +
|
||||
", date=" + date +
|
||||
'}';
|
||||
|
|
|
|||
|
|
@ -4,64 +4,72 @@ import com.fasterxml.jackson.annotation.JsonInclude;
|
|||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
|
||||
//import javax.persistence.Column;
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
|
||||
//@Entity
|
||||
//@Table(name = "payload")
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
@JsonPropertyOrder({
|
||||
"id",
|
||||
"original_url",
|
||||
"actual_url",
|
||||
"date_acquired",
|
||||
"date",
|
||||
"mime_type",
|
||||
"size",
|
||||
"more_info",
|
||||
"md5",
|
||||
"hash",
|
||||
"location",
|
||||
"provenance"
|
||||
})
|
||||
public class Payload {
|
||||
public class Payload implements Serializable {
|
||||
|
||||
//@Column(name = "id")
|
||||
@JsonProperty("id")
|
||||
private String id;
|
||||
|
||||
//@Column(name = "original_url")
|
||||
@JsonProperty("original_url")
|
||||
private String original_url;
|
||||
|
||||
//@Column(name = "date")
|
||||
@JsonProperty("actual_url")
|
||||
private String actual_url;
|
||||
|
||||
@JsonProperty("date_acquired")
|
||||
//@Column(name = "date")
|
||||
@JsonProperty("date")
|
||||
private Date date_acquired;
|
||||
|
||||
//@Column(name = "mimetype")
|
||||
@JsonProperty("mime_type")
|
||||
private String mime_type;
|
||||
|
||||
//@Column(name = "size")
|
||||
@JsonProperty("size")
|
||||
private Long size; // In bytes.
|
||||
|
||||
@JsonProperty("more_info")
|
||||
private String more_info;
|
||||
|
||||
@JsonProperty("md5")
|
||||
private String md5;
|
||||
//@Column(name = "hash")
|
||||
@JsonProperty("hash")
|
||||
private String hash;
|
||||
|
||||
//@Column(name = "location")
|
||||
@JsonProperty("location")
|
||||
private String location;
|
||||
|
||||
//@Column(name = "provenance")
|
||||
@JsonProperty("provenance")
|
||||
private String provenance;
|
||||
private String provenance; // "crawl:<PluginName>"
|
||||
|
||||
|
||||
public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String more_info, String md5, String location, String provenance) {
|
||||
public Payload(String id, String original_url, String actual_url, Date date_acquired, String mime_type, Long size, String hash, String location, String provenance) {
|
||||
this.id = id;
|
||||
this.original_url = original_url;
|
||||
this.actual_url = actual_url;
|
||||
this.date_acquired = date_acquired;
|
||||
this.mime_type = mime_type;
|
||||
this.size = size;
|
||||
this.more_info = more_info;
|
||||
this.md5 = md5;
|
||||
this.hash = hash;
|
||||
this.location = location;
|
||||
this.provenance = provenance;
|
||||
}
|
||||
|
|
@ -114,20 +122,12 @@ public class Payload {
|
|||
this.size = size;
|
||||
}
|
||||
|
||||
public String getMore_info() {
|
||||
return more_info;
|
||||
public String getHash() {
|
||||
return hash;
|
||||
}
|
||||
|
||||
public void setMore_info(String more_info) {
|
||||
this.more_info = more_info;
|
||||
}
|
||||
|
||||
public String getMd5() {
|
||||
return md5;
|
||||
}
|
||||
|
||||
public void setMd5(String md5) {
|
||||
this.md5 = md5;
|
||||
public void setHash(String hash) {
|
||||
this.hash = hash;
|
||||
}
|
||||
|
||||
public String getLocation() {
|
||||
|
|
@ -155,8 +155,7 @@ public class Payload {
|
|||
", date_acquired='" + date_acquired + '\'' +
|
||||
", mime_type='" + mime_type + '\'' +
|
||||
", size='" + size + '\'' +
|
||||
", more_info='" + more_info + '\'' +
|
||||
", md5='" + md5 + '\'' +
|
||||
", md5='" + hash + '\'' +
|
||||
", location='" + location + '\'' +
|
||||
", provenance='" + provenance + '\'' +
|
||||
'}';
|
||||
|
|
|
|||
|
|
@ -19,15 +19,15 @@ public class WorkerReport {
|
|||
@JsonProperty("workerId")
|
||||
private String workerId;
|
||||
|
||||
@JsonProperty("assignmentId")
|
||||
private int assignmentId;
|
||||
@JsonProperty("assignmentRequestCounter")
|
||||
private Long assignmentRequestCounter;
|
||||
|
||||
@JsonProperty("urlReports")
|
||||
private List<UrlReport> urlReports;
|
||||
|
||||
public WorkerReport(String workerId, int assignmentId, List<UrlReport> urlReports) {
|
||||
public WorkerReport(String workerId, Long assignmentRequestCounter, List<UrlReport> urlReports) {
|
||||
this.workerId = workerId;
|
||||
this.assignmentId = assignmentId;
|
||||
this.assignmentRequestCounter = assignmentRequestCounter;
|
||||
this.urlReports = urlReports;
|
||||
}
|
||||
|
||||
|
|
@ -39,12 +39,12 @@ public class WorkerReport {
|
|||
this.workerId = workerId;
|
||||
}
|
||||
|
||||
public int getAssignmentId() {
|
||||
return this.assignmentId;
|
||||
public Long getAssignmentRequestCounter() {
|
||||
return assignmentRequestCounter;
|
||||
}
|
||||
|
||||
public void setAssignmentId(int assignmentId) {
|
||||
this.assignmentId = assignmentId;
|
||||
public void setAssignmentRequestCounter(Long assignmentRequestCounter) {
|
||||
this.assignmentRequestCounter = assignmentRequestCounter;
|
||||
}
|
||||
|
||||
public List<UrlReport> getUrlReports() {
|
||||
|
|
@ -59,7 +59,7 @@ public class WorkerReport {
|
|||
public String toString() {
|
||||
return "WorkerReport{" +
|
||||
"workerId='" + workerId + '\'' +
|
||||
", assignmentId=" + assignmentId +
|
||||
", assignmentRequestCounter=" + assignmentRequestCounter +
|
||||
", urlReports=" + urlReports +
|
||||
'}';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,29 +4,44 @@ import com.fasterxml.jackson.annotation.JsonInclude;
|
|||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import eu.openaire.urls_controller.models.Assignment;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class AssignmentResponse {
|
||||
|
||||
@JsonProperty("assignment")
|
||||
private Assignment assignment;
|
||||
@JsonProperty("assignmentCounter")
|
||||
private Long assignmentCounter;
|
||||
|
||||
public AssignmentResponse(Assignment assignment) {
|
||||
this.assignment = assignment;
|
||||
@JsonProperty("assignments")
|
||||
private List<Assignment> assignments;
|
||||
|
||||
public AssignmentResponse(Long assignmentCounter, List<Assignment> assignments) {
|
||||
this.assignmentCounter = assignmentCounter;
|
||||
this.assignments = assignments;
|
||||
}
|
||||
|
||||
public Assignment getAssignment() {
|
||||
return assignment;
|
||||
public Long getAssignmentCounter() {
|
||||
return assignmentCounter;
|
||||
}
|
||||
|
||||
public void setAssignment(Assignment assignment) {
|
||||
this.assignment = assignment;
|
||||
public void setAssignmentCounter(Long assignmentCounter) {
|
||||
this.assignmentCounter = assignmentCounter;
|
||||
}
|
||||
|
||||
public List<Assignment> getAssignments() {
|
||||
return assignments;
|
||||
}
|
||||
|
||||
public void setAssignments(List<Assignment> assignments) {
|
||||
this.assignments = assignments;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AssignmentResponse{" +
|
||||
"assignment=" + assignment +
|
||||
"assignments=" + assignments +
|
||||
'}';
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
package eu.openaire.urls_controller.util;
|
||||
|
||||
public interface ControllerConstants {
|
||||
|
||||
int ASSIGNMENTS_LIMIT = 10000; // The general assignments-limit the Controller will get. If the worker cannot handle them, then the worker's limit will be applied.
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue