2021-03-16 14:25:15 +01:00
package eu.openaire.urls_controller.controllers ;
2021-05-18 16:23:20 +02:00
import com.google.common.collect.HashMultimap ;
import eu.openaire.urls_controller.models.Assignment ;
2021-05-20 01:50:50 +02:00
import eu.openaire.urls_controller.models.Datasource ;
2021-05-18 16:23:20 +02:00
import eu.openaire.urls_controller.models.Task ;
2021-06-22 04:38:48 +02:00
import eu.openaire.urls_controller.payloads.requests.WorkerReport ;
2021-05-18 16:23:20 +02:00
import eu.openaire.urls_controller.payloads.responces.AssignmentResponse ;
2021-07-05 13:04:39 +02:00
//import eu.openaire.urls_controller.repositories.AssignmentRepository;
import eu.openaire.urls_controller.util.ControllerConstants ;
2021-05-18 16:23:20 +02:00
import eu.openaire.urls_controller.util.FileUtils ;
2021-05-20 01:50:50 +02:00
import eu.openaire.urls_controller.util.GenericUtils ;
2021-03-16 14:25:15 +01:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
2021-06-22 04:38:48 +02:00
import org.springframework.http.HttpStatus ;
2021-03-16 14:25:15 +01:00
import org.springframework.http.ResponseEntity ;
2021-06-22 04:38:48 +02:00
import org.springframework.web.bind.annotation.* ;
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
import java.util.* ;
2021-07-05 13:04:39 +02:00
import java.util.concurrent.atomic.AtomicLong ;
2021-03-16 14:25:15 +01:00
@RestController
@RequestMapping ( " /urls " )
public class UrlController {
private static final Logger logger = LoggerFactory . getLogger ( UrlController . class ) ;
2021-08-05 14:43:37 +02:00
private static AtomicLong assignmentCounter = new AtomicLong ( 0 ) ; // Just for the "getTestUrls"-endpoint.
2021-03-16 14:25:15 +01:00
}
@GetMapping ( " " )
2021-08-05 14:43:37 +02:00
public ResponseEntity < ? > getUrls ( @RequestParam String workerId , @RequestParam int workerAssignmentsLimit ) {
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
List < Task > tasks = new ArrayList < > ( ) ;
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
// TODO - Create the Assignment from the id-urls stored in the database up to the tasks-limit.
2021-08-05 14:43:37 +02:00
// TODO - Make sure the Date is the same for all entries!
Date date = new Date ( ) ; // Store it here, in order to have the same for sure.
int assignmentsLimit = ControllerConstants . ASSIGNMENTS_LIMIT ;
if ( assignmentsLimit > workerAssignmentsLimit )
assignmentsLimit = workerAssignmentsLimit ;
2021-03-16 14:25:15 +01:00
2021-07-05 13:04:39 +02:00
int tasksLimitForAssignment = ControllerConstants . ASSIGNMENTS_LIMIT ;
if ( tasksLimitForAssignment > workerTasksLimit )
tasksLimitForAssignment = workerTasksLimit ;
List < Assignment > assignments = null ; // TODO -> // assignmentRepository.getNewAssignments(tasksLimitForAssignment);
//Assignment assignment = new Assignment(assignmentId, tasks, workerId, date);
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
// TODO - Write the Assignment details to the database and then send it to the worker.
2021-03-16 14:25:15 +01:00
2021-08-05 14:43:37 +02:00
logger . info ( " Sending assignment_ " + assignmentCounter . incrementAndGet ( ) + " to worker with ID: " + workerId ) ;
2021-06-10 13:21:39 +02:00
2021-08-05 14:43:37 +02:00
return ResponseEntity . status ( 200 ) . header ( " Content-Type " , " application/json " ) . body ( new AssignmentResponse ( assignmentCounter . get ( ) , assignments ) ) ;
2021-03-16 14:25:15 +01:00
}
2021-06-22 04:38:48 +02:00
@PostMapping ( " addWorkerReport " )
public ResponseEntity < ? > addWorkerReport ( @RequestBody WorkerReport workerReport ) {
if ( workerReport = = null )
return ResponseEntity . status ( HttpStatus . BAD_REQUEST ) . build ( ) ;
logger . debug ( " Received the WorkerReport: \ n " + workerReport . toString ( ) ) ;
// TODO - Store the workerReport into the database.
return ResponseEntity . status ( HttpStatus . OK ) . build ( ) ;
}
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
@GetMapping ( " test " )
2021-08-05 14:43:37 +02:00
public ResponseEntity < ? > getTestUrls ( @RequestParam String workerId , @RequestParam int workerAssignmentsLimit ) {
2021-03-16 14:25:15 +01:00
2021-06-10 13:21:39 +02:00
try {
new FileUtils ( ) ; // Find the input file.
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
2021-07-05 13:04:39 +02:00
return ResponseEntity . status ( 500 ) . body ( " The resource file, for the requested assignments, was not found. " ) ;
2021-06-10 13:21:39 +02:00
}
2021-03-16 14:25:15 +01:00
2021-07-05 13:04:39 +02:00
List < Assignment > assignments = new ArrayList < > ( ) ;
2021-05-18 16:23:20 +02:00
HashMultimap < String , String > loadedIdUrlPairs ;
boolean isFirstRun = true ;
2021-07-05 13:04:39 +02:00
boolean assignmentsLimitReached = false ;
Date date = new Date ( ) ;
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
// Start loading urls.
2021-06-10 13:21:39 +02:00
while ( true ) {
2021-05-18 16:23:20 +02:00
loadedIdUrlPairs = FileUtils . getNextIdUrlPairBatchFromJson ( ) ; // Take urls from jsonFile.
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
if ( FileUtils . isFinishedLoading ( loadedIdUrlPairs . isEmpty ( ) , isFirstRun ) ) // Throws RuntimeException which is automatically passed on.
break ;
else
isFirstRun = false ;
2021-03-16 14:25:15 +01:00
2021-05-18 16:23:20 +02:00
Set < Map . Entry < String , String > > pairs = loadedIdUrlPairs . entries ( ) ;
for ( Map . Entry < String , String > pair : pairs )
{
2021-08-05 14:43:37 +02:00
if ( assignments . size ( ) > = workerAssignmentsLimit ) {
2021-07-05 13:04:39 +02:00
assignmentsLimitReached = true ;
2021-05-18 16:23:20 +02:00
break ;
}
2021-05-20 01:50:50 +02:00
int randomNum = GenericUtils . getRandomNumber ( 1 , 5 ) ;
2021-07-05 13:04:39 +02:00
assignments . add ( new Assignment ( pair . getKey ( ) , pair . getValue ( ) , new Datasource ( " ID_ " + randomNum , " NAME_ " + randomNum ) , workerId , date ) ) ;
2021-05-18 16:23:20 +02:00
} // end pairs-for-loop
2021-07-05 13:04:39 +02:00
if ( assignmentsLimitReached ) {
2021-08-05 14:43:37 +02:00
logger . debug ( " Done loading urls from the inputFile as the assignmentsLimit ( " + workerAssignmentsLimit + " ) was reached. " ) ;
2021-05-18 16:23:20 +02:00
break ;
}
} // end loading-while-loop
2021-06-10 19:24:51 +02:00
if ( FileUtils . inputScanner . get ( ) ! = null ) // Check if the initial value is null.
FileUtils . inputScanner . get ( ) . close ( ) ;
2021-06-10 13:21:39 +02:00
2021-08-05 14:43:37 +02:00
logger . info ( " Sending AssignmentResponse_ " + assignmentCounter . incrementAndGet ( ) + " with " + assignments . size ( ) + " assignments ( " + FileUtils . duplicateIdUrlEntries . get ( ) + " more assignments were discarded as duplicates), to worker with ID: " + workerId ) ;
2021-06-10 13:21:39 +02:00
2021-08-05 14:43:37 +02:00
return ResponseEntity . status ( 200 ) . header ( " Content-Type " , " application/json " ) . body ( new AssignmentResponse ( assignmentCounter . get ( ) , assignments ) ) ;
2021-05-18 16:23:20 +02:00
}
2021-03-16 14:25:15 +01:00
}