You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
106 lines
4.0 KiB
Java
106 lines
4.0 KiB
Java
package eu.openaire.urls_controller.controllers;
|
|
|
|
import com.google.common.collect.HashMultimap;
|
|
import eu.openaire.urls_controller.models.Assignment;
|
|
import eu.openaire.urls_controller.models.Datasource;
|
|
import eu.openaire.urls_controller.models.Task;
|
|
import eu.openaire.urls_controller.payloads.responces.AssignmentResponse;
|
|
import eu.openaire.urls_controller.util.FileUtils;
|
|
import eu.openaire.urls_controller.util.GenericUtils;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.http.ResponseEntity;
|
|
import org.springframework.web.bind.annotation.GetMapping;
|
|
import org.springframework.web.bind.annotation.RequestMapping;
|
|
import org.springframework.web.bind.annotation.RequestParam;
|
|
import org.springframework.web.bind.annotation.RestController;
|
|
|
|
import java.util.*;
|
|
|
|
@RestController
|
|
@RequestMapping("/urls")
|
|
public class UrlController {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(UrlController.class);
|
|
|
|
|
|
public UrlController() {
|
|
|
|
}
|
|
|
|
|
|
@GetMapping("")
|
|
public ResponseEntity<?> getUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
|
|
|
|
List<Task> tasks = new ArrayList<>();
|
|
|
|
// TODO - Create the Assignment from the id-urls stored in the database up to the tasks-limit.
|
|
int assignmentId = 0;
|
|
|
|
Assignment assignment = new Assignment(assignmentId, tasks, workerId, new Date());
|
|
|
|
// TODO - Write the Assignment details to the database and then send it to the worker.
|
|
|
|
logger.info("Sending assignment_" + assignment.getAssignmentId() + " to worker with ID: " + workerId);
|
|
|
|
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
|
|
}
|
|
|
|
|
|
private static int assignmentId = -1; // Just for the "getTestUrls"-endpoint.
|
|
|
|
@GetMapping("test")
|
|
public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int tasksLimit) {
|
|
|
|
try {
|
|
new FileUtils(); // Find the input file.
|
|
} catch (Exception e) {
|
|
logger.error(e.getMessage());
|
|
return ResponseEntity.status(500).body("The resource file for the requested tasks was not found.");
|
|
}
|
|
|
|
List<Task> tasks = new ArrayList<>();
|
|
HashMultimap<String, String> loadedIdUrlPairs;
|
|
boolean isFirstRun = true;
|
|
boolean tasksLimitReached = false;
|
|
|
|
// Start loading urls.
|
|
while ( true ) {
|
|
loadedIdUrlPairs = FileUtils.getNextIdUrlPairBatchFromJson(); // Take urls from jsonFile.
|
|
|
|
if ( FileUtils.isFinishedLoading(loadedIdUrlPairs.isEmpty(), isFirstRun) ) // Throws RuntimeException which is automatically passed on.
|
|
break;
|
|
else
|
|
isFirstRun = false;
|
|
|
|
Set<Map.Entry<String, String>> pairs = loadedIdUrlPairs.entries();
|
|
|
|
for ( Map.Entry<String,String> pair : pairs )
|
|
{
|
|
if ( tasks.size() >= tasksLimit ) {
|
|
tasksLimitReached = true;
|
|
break;
|
|
}
|
|
|
|
int randomNum = GenericUtils.getRandomNumber(1, 5);
|
|
tasks.add(new Task(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum)));
|
|
}// end pairs-for-loop
|
|
|
|
if ( tasksLimitReached ) {
|
|
logger.debug("Done loading urls from the inputFile as the tasksLimit (" + tasksLimit + ") was reached.");
|
|
break;
|
|
}
|
|
}// end loading-while-loop
|
|
|
|
if ( FileUtils.inputScanner != null )
|
|
FileUtils.inputScanner.close();
|
|
|
|
Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date());
|
|
|
|
logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries + " more tasks were discarded as duplicates), to worker with ID: " + workerId);
|
|
|
|
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
|
|
}
|
|
|
|
}
|