Allow handling of concurrent requests to the "getTestUrls"-endpoint.

This commit is contained in:
Lampros Smyrnaios 2021-06-10 20:24:51 +03:00
parent 308cab5ecd
commit c194af167f
2 changed files with 22 additions and 20 deletions

View File

@ -92,12 +92,12 @@ public class UrlController {
} }
}// end loading-while-loop }// end loading-while-loop
if ( FileUtils.inputScanner != null ) if ( FileUtils.inputScanner.get() != null ) // Check if the initial value is null.
FileUtils.inputScanner.close(); FileUtils.inputScanner.get().close();
Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date()); Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date());
logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries + " more tasks were discarded as duplicates), to worker with ID: " + workerId); logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries.get() + " more tasks were discarded as duplicates), to worker with ID: " + workerId);
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment)); return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
} }

View File

@ -18,14 +18,14 @@ public class FileUtils {
private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
public static Scanner inputScanner = null; public static ThreadLocal<Scanner> inputScanner = new ThreadLocal<Scanner>(); // Every Thread has its own variable.
private static int fileIndex = 0; private static ThreadLocal<Integer> fileIndex = new ThreadLocal<Integer>();
private static int unretrievableInputLines = 0; private static ThreadLocal<Integer> unretrievableInputLines = new ThreadLocal<Integer>();
public static int duplicateIdUrlEntries = 0; public static ThreadLocal<Integer> duplicateIdUrlEntries = new ThreadLocal<Integer>();
public static int jsonBatchSize = 3000; public static final int jsonBatchSize = 3000;
private static final String utf8Charset = "UTF-8"; private static final String utf8Charset = "UTF-8";
public static String inputFileFullPath; public static String inputFileFullPath;
private static String workingDir = System.getProperty("user.dir") + File.separator; private static final String workingDir = System.getProperty("user.dir") + File.separator;
public FileUtils() throws RuntimeException public FileUtils() throws RuntimeException
@ -39,8 +39,10 @@ public class FileUtils {
logger.debug("Going to retrieve the data from the inputResourceFile: " + resourceFileName); logger.debug("Going to retrieve the data from the inputResourceFile: " + resourceFileName);
FileUtils.inputScanner = new Scanner(inputStream, utf8Charset); FileUtils.inputScanner.set(new Scanner(inputStream, utf8Charset));
fileIndex = 0; // Re-initialize the file-number-pointer. fileIndex.set(0); // Re-initialize the file-number-pointer.
unretrievableInputLines.set(0);
duplicateIdUrlEntries.set(0);
} }
@ -100,32 +102,32 @@ public class FileUtils {
HashMultimap<String, String> idAndUrlMappedInput = HashMultimap.create(expectedIDsPerBatch, expectedPathsPerID); HashMultimap<String, String> idAndUrlMappedInput = HashMultimap.create(expectedIDsPerBatch, expectedPathsPerID);
int curBeginning = fileIndex; int curBeginning = fileIndex.get();
while ( inputScanner.hasNextLine() && (fileIndex < (curBeginning + jsonBatchSize)) ) while ( inputScanner.get().hasNextLine() && (fileIndex.get() < (curBeginning + jsonBatchSize)) )
{// While (!EOF) and inside the current url-batch, iterate through lines. {// While (!EOF) and inside the current url-batch, iterate through lines.
//logger.debug("fileIndex: " + FileUtils.fileIndex); // DEBUG! //logger.debug("fileIndex: " + FileUtils.fileIndex.get()); // DEBUG!
// Take each line, remove potential double quotes. // Take each line, remove potential double quotes.
String retrievedLineStr = inputScanner.nextLine(); String retrievedLineStr = inputScanner.get().nextLine();
//logger.debug("Loaded from inputFile: " + retrievedLineStr); // DEBUG! //logger.debug("Loaded from inputFile: " + retrievedLineStr); // DEBUG!
fileIndex ++; fileIndex.set(fileIndex.get() +1);
if ( retrievedLineStr.isEmpty() ) { if ( retrievedLineStr.isEmpty() ) {
unretrievableInputLines ++; unretrievableInputLines.set(unretrievableInputLines.get() +1);
continue; continue;
} }
if ( (inputIdUrlTuple = jsonDecoder(retrievedLineStr)) == null ) { // Decode the jsonLine and take the two attributes. if ( (inputIdUrlTuple = jsonDecoder(retrievedLineStr)) == null ) { // Decode the jsonLine and take the two attributes.
logger.warn("A problematic inputLine found: \t" + retrievedLineStr); logger.warn("A problematic inputLine found: \t" + retrievedLineStr);
unretrievableInputLines ++; unretrievableInputLines.set(unretrievableInputLines.get() +1);
continue; continue;
} }
if ( !idAndUrlMappedInput.put(inputIdUrlTuple.getId(), inputIdUrlTuple.getUrl()) ) { // We have a duplicate url in the input.. log it here as we cannot pass it through the HashMultimap. It's possible that this as well as the original might be/give a docUrl. if ( !idAndUrlMappedInput.put(inputIdUrlTuple.getId(), inputIdUrlTuple.getUrl()) ) { // We have a duplicate url in the input.. log it here as we cannot pass it through the HashMultimap. It's possible that this as well as the original might be/give a docUrl.
duplicateIdUrlEntries ++; duplicateIdUrlEntries.set(duplicateIdUrlEntries.get() +1);
} }
} }
@ -139,7 +141,7 @@ public class FileUtils {
*/ */
public static int getCurrentlyLoadedUrls() // In the end, it gives the total number of urls we have processed. public static int getCurrentlyLoadedUrls() // In the end, it gives the total number of urls we have processed.
{ {
return FileUtils.fileIndex - FileUtils.unretrievableInputLines; return FileUtils.fileIndex.get() - FileUtils.unretrievableInputLines.get();
} }