forked from lsmyrnaios/UrlsController
Allow handling of concurrent requests to the "getTestUrls"-endpoint.
This commit is contained in:
parent
308cab5ecd
commit
c194af167f
|
@ -92,12 +92,12 @@ public class UrlController {
|
||||||
}
|
}
|
||||||
}// end loading-while-loop
|
}// end loading-while-loop
|
||||||
|
|
||||||
if ( FileUtils.inputScanner != null )
|
if ( FileUtils.inputScanner.get() != null ) // Check if the initial value is null.
|
||||||
FileUtils.inputScanner.close();
|
FileUtils.inputScanner.get().close();
|
||||||
|
|
||||||
Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date());
|
Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date());
|
||||||
|
|
||||||
logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries + " more tasks were discarded as duplicates), to worker with ID: " + workerId);
|
logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries.get() + " more tasks were discarded as duplicates), to worker with ID: " + workerId);
|
||||||
|
|
||||||
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
|
return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment));
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,14 +18,14 @@ public class FileUtils {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
|
private static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
|
||||||
|
|
||||||
public static Scanner inputScanner = null;
|
public static ThreadLocal<Scanner> inputScanner = new ThreadLocal<Scanner>(); // Every Thread has its own variable.
|
||||||
private static int fileIndex = 0;
|
private static ThreadLocal<Integer> fileIndex = new ThreadLocal<Integer>();
|
||||||
private static int unretrievableInputLines = 0;
|
private static ThreadLocal<Integer> unretrievableInputLines = new ThreadLocal<Integer>();
|
||||||
public static int duplicateIdUrlEntries = 0;
|
public static ThreadLocal<Integer> duplicateIdUrlEntries = new ThreadLocal<Integer>();
|
||||||
public static int jsonBatchSize = 3000;
|
public static final int jsonBatchSize = 3000;
|
||||||
private static final String utf8Charset = "UTF-8";
|
private static final String utf8Charset = "UTF-8";
|
||||||
public static String inputFileFullPath;
|
public static String inputFileFullPath;
|
||||||
private static String workingDir = System.getProperty("user.dir") + File.separator;
|
private static final String workingDir = System.getProperty("user.dir") + File.separator;
|
||||||
|
|
||||||
|
|
||||||
public FileUtils() throws RuntimeException
|
public FileUtils() throws RuntimeException
|
||||||
|
@ -39,8 +39,10 @@ public class FileUtils {
|
||||||
|
|
||||||
logger.debug("Going to retrieve the data from the inputResourceFile: " + resourceFileName);
|
logger.debug("Going to retrieve the data from the inputResourceFile: " + resourceFileName);
|
||||||
|
|
||||||
FileUtils.inputScanner = new Scanner(inputStream, utf8Charset);
|
FileUtils.inputScanner.set(new Scanner(inputStream, utf8Charset));
|
||||||
fileIndex = 0; // Re-initialize the file-number-pointer.
|
fileIndex.set(0); // Re-initialize the file-number-pointer.
|
||||||
|
unretrievableInputLines.set(0);
|
||||||
|
duplicateIdUrlEntries.set(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -100,32 +102,32 @@ public class FileUtils {
|
||||||
|
|
||||||
HashMultimap<String, String> idAndUrlMappedInput = HashMultimap.create(expectedIDsPerBatch, expectedPathsPerID);
|
HashMultimap<String, String> idAndUrlMappedInput = HashMultimap.create(expectedIDsPerBatch, expectedPathsPerID);
|
||||||
|
|
||||||
int curBeginning = fileIndex;
|
int curBeginning = fileIndex.get();
|
||||||
|
|
||||||
while ( inputScanner.hasNextLine() && (fileIndex < (curBeginning + jsonBatchSize)) )
|
while ( inputScanner.get().hasNextLine() && (fileIndex.get() < (curBeginning + jsonBatchSize)) )
|
||||||
{// While (!EOF) and inside the current url-batch, iterate through lines.
|
{// While (!EOF) and inside the current url-batch, iterate through lines.
|
||||||
|
|
||||||
//logger.debug("fileIndex: " + FileUtils.fileIndex); // DEBUG!
|
//logger.debug("fileIndex: " + FileUtils.fileIndex.get()); // DEBUG!
|
||||||
|
|
||||||
// Take each line, remove potential double quotes.
|
// Take each line, remove potential double quotes.
|
||||||
String retrievedLineStr = inputScanner.nextLine();
|
String retrievedLineStr = inputScanner.get().nextLine();
|
||||||
//logger.debug("Loaded from inputFile: " + retrievedLineStr); // DEBUG!
|
//logger.debug("Loaded from inputFile: " + retrievedLineStr); // DEBUG!
|
||||||
|
|
||||||
fileIndex ++;
|
fileIndex.set(fileIndex.get() +1);
|
||||||
|
|
||||||
if ( retrievedLineStr.isEmpty() ) {
|
if ( retrievedLineStr.isEmpty() ) {
|
||||||
unretrievableInputLines ++;
|
unretrievableInputLines.set(unretrievableInputLines.get() +1);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( (inputIdUrlTuple = jsonDecoder(retrievedLineStr)) == null ) { // Decode the jsonLine and take the two attributes.
|
if ( (inputIdUrlTuple = jsonDecoder(retrievedLineStr)) == null ) { // Decode the jsonLine and take the two attributes.
|
||||||
logger.warn("A problematic inputLine found: \t" + retrievedLineStr);
|
logger.warn("A problematic inputLine found: \t" + retrievedLineStr);
|
||||||
unretrievableInputLines ++;
|
unretrievableInputLines.set(unretrievableInputLines.get() +1);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( !idAndUrlMappedInput.put(inputIdUrlTuple.getId(), inputIdUrlTuple.getUrl()) ) { // We have a duplicate url in the input.. log it here as we cannot pass it through the HashMultimap. It's possible that this as well as the original might be/give a docUrl.
|
if ( !idAndUrlMappedInput.put(inputIdUrlTuple.getId(), inputIdUrlTuple.getUrl()) ) { // We have a duplicate url in the input.. log it here as we cannot pass it through the HashMultimap. It's possible that this as well as the original might be/give a docUrl.
|
||||||
duplicateIdUrlEntries ++;
|
duplicateIdUrlEntries.set(duplicateIdUrlEntries.get() +1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -139,7 +141,7 @@ public class FileUtils {
|
||||||
*/
|
*/
|
||||||
public static int getCurrentlyLoadedUrls() // In the end, it gives the total number of urls we have processed.
|
public static int getCurrentlyLoadedUrls() // In the end, it gives the total number of urls we have processed.
|
||||||
{
|
{
|
||||||
return FileUtils.fileIndex - FileUtils.unretrievableInputLines;
|
return FileUtils.fileIndex.get() - FileUtils.unretrievableInputLines.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue