forked from lsmyrnaios/UrlsController
85 lines
3.9 KiB
Java
85 lines
3.9 KiB
Java
package eu.openaire.urls_controller.controllers;
|
|
|
|
import com.google.common.collect.HashMultimap;
|
|
import eu.openaire.urls_controller.models.Assignment;
|
|
import eu.openaire.urls_controller.models.Datasource;
|
|
import eu.openaire.urls_controller.payloads.responces.AssignmentsResponse;
|
|
import eu.openaire.urls_controller.util.GenericUtils;
|
|
import eu.openaire.urls_controller.util.TestFileUtils;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.http.HttpStatus;
|
|
import org.springframework.http.ResponseEntity;
|
|
import org.springframework.web.bind.annotation.GetMapping;
|
|
import org.springframework.web.bind.annotation.RequestParam;
|
|
import org.springframework.web.bind.annotation.RequestMapping;
|
|
import org.springframework.web.bind.annotation.RestController;
|
|
|
|
import java.sql.Timestamp;
|
|
import java.util.*;
|
|
import java.util.concurrent.atomic.AtomicLong;
|
|
|
|
@RestController
|
|
@RequestMapping("/test")
|
|
public class TestController extends GeneralController {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(TestController.class);
|
|
|
|
@Autowired
|
|
private TestFileUtils fileUtils;
|
|
|
|
@Value("${services.pdfaggregation.controller.assignmentLimit}")
|
|
private int assignmentLimit;
|
|
|
|
private static final AtomicLong assignmentsBatchCounter = new AtomicLong(0); // Just for the "getTestUrls"-endpoint.
|
|
|
|
@GetMapping("test")
|
|
public ResponseEntity<?> getTestUrls(@RequestParam String workerId, @RequestParam int workerAssignmentsLimit) {
|
|
|
|
logger.info("Worker with id: \"" + workerId + "\", requested " + workerAssignmentsLimit + " test-assignments. The assignments-limit of the controller is: " + this.assignmentLimit);
|
|
|
|
List<Assignment> assignments = new ArrayList<>();
|
|
HashMultimap<String, String> loadedIdUrlPairs;
|
|
boolean isFirstRun = true;
|
|
boolean assignmentsLimitReached = false;
|
|
Timestamp timestamp = new Timestamp(System.currentTimeMillis()); // Store it here, in order to have the same for all current records.
|
|
|
|
// Start loading urls.
|
|
while ( true ) {
|
|
loadedIdUrlPairs = fileUtils.getNextIdUrlPairBatchFromJson(); // Take urls from jsonFile.
|
|
|
|
if ( fileUtils.isFinishedLoading(loadedIdUrlPairs.isEmpty(), isFirstRun) ) // Throws RuntimeException which is automatically passed on.
|
|
break;
|
|
else
|
|
isFirstRun = false;
|
|
|
|
Set<Map.Entry<String, String>> pairs = loadedIdUrlPairs.entries();
|
|
|
|
for ( Map.Entry<String,String> pair : pairs ) {
|
|
if ( assignments.size() >= workerAssignmentsLimit ) {
|
|
assignmentsLimitReached = true;
|
|
break;
|
|
}
|
|
|
|
int randomNum = GenericUtils.getRandomNumber(1, 5);
|
|
assignments.add(new Assignment(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum), workerId, timestamp));
|
|
}// end pairs-for-loop
|
|
|
|
if ( assignmentsLimitReached ) {
|
|
logger.debug("Done loading urls from the inputFile as the assignmentsLimit (" + workerAssignmentsLimit + ") was reached.");
|
|
break;
|
|
}
|
|
}// end loading-while-loop
|
|
|
|
Scanner scanner = fileUtils.inputScanner.get();
|
|
if ( scanner != null ) // Check if the initial value is null.
|
|
scanner.close();
|
|
|
|
long curAssignmentsBatchCounter = assignmentsBatchCounter.incrementAndGet();
|
|
logger.info("Sending batch_" + curAssignmentsBatchCounter + " with " + assignments.size() + " assignments (" + fileUtils.duplicateIdUrlEntries.get() + " more assignments were discarded as duplicates), to worker with ID: " + workerId);
|
|
return ResponseEntity.status(HttpStatus.OK).header("Content-Type", "application/json").body(new AssignmentsResponse(curAssignmentsBatchCounter, assignments));
|
|
}
|
|
}
|