From 308cab5ecd77f084b4bea7c6028f7301ec9fd481 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Thu, 10 Jun 2021 14:21:39 +0300 Subject: [PATCH] - Return an HTTP-500-error when the server cannot find the resourceFile requested by the "getTestUrls"-endpoint. - Close the "inputScanner" after each use when retrieving the test-tasks. - Show info-logs when sending an assignment to a worker. - Code cleanup. --- build.gradle | 5 +---- .../controllers/UrlController.java | 22 ++++++++++++++----- .../urls_controller/util/FileUtils.java | 17 +++++++------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/build.gradle b/build.gradle index f9ebc1f..bf0a885 100644 --- a/build.gradle +++ b/build.gradle @@ -28,16 +28,13 @@ dependencies { implementation("org.springframework.security:spring-security-core:${springSecurityVersion}") implementation("org.springframework.security:spring-security-web:${springSecurityVersion}") implementation("org.springframework.security:spring-security-config:${springSecurityVersion}") - implementation("io.jsonwebtoken:jjwt:0.9.1") + //implementation("io.jsonwebtoken:jjwt:0.9.1") // Use this in case we use auth-tokens later on. // https://mvnrepository.com/artifact/com.google.guava/guava implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' implementation "org.projectlombok:lombok:1.18.20" - implementation 'com.google.code.gson:gson:2.8.7' implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' - implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' - implementation group: 'commons-io', name: 'commons-io', version: '2.9.0' testImplementation group: 'org.springframework.security', name: 'spring-security-test', version: springSecurityVersion testImplementation 'org.springframework.boot:spring-boot-starter-test' diff --git a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java index 15d0d1c..2295c25 100644 --- a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java +++ b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java @@ -41,6 +41,8 @@ public class UrlController { // TODO - Write the Assignment details to the database and then send it to the worker. + logger.info("Sending assignment_" + assignment.getAssignmentId() + " to worker with ID: " + workerId); + return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment)); } @@ -50,16 +52,20 @@ public class UrlController { @GetMapping("test") public ResponseEntity getTestUrls(@RequestParam String workerId, @RequestParam int tasksLimit) { - List tasks = new ArrayList<>(); - new FileUtils(); // Find the input file. + try { + new FileUtils(); // Find the input file. + } catch (Exception e) { + logger.error(e.getMessage()); + return ResponseEntity.status(500).body("The resource file for the requested tasks was not found."); + } + List tasks = new ArrayList<>(); HashMultimap loadedIdUrlPairs; boolean isFirstRun = true; boolean tasksLimitReached = false; // Start loading urls. - while ( true ) - { + while ( true ) { loadedIdUrlPairs = FileUtils.getNextIdUrlPairBatchFromJson(); // Take urls from jsonFile. if ( FileUtils.isFinishedLoading(loadedIdUrlPairs.isEmpty(), isFirstRun) ) // Throws RuntimeException which is automatically passed on. @@ -81,14 +87,18 @@ public class UrlController { }// end pairs-for-loop if ( tasksLimitReached ) { - logger.debug("Done loading tasksLimit (" + tasksLimit + ") urls from the inputFile."); + logger.debug("Done loading urls from the inputFile as the tasksLimit (" + tasksLimit + ") was reached."); break; } - }// end loading-while-loop + if ( FileUtils.inputScanner != null ) + FileUtils.inputScanner.close(); + Assignment assignment = new Assignment((++assignmentId), tasks, workerId, new Date()); + logger.info("Sending assignment_" + assignment.getAssignmentId() + " with " + tasks.size() + " tasks (" + FileUtils.duplicateIdUrlEntries + " more tasks were discarded as duplicates), to worker with ID: " + workerId); + return ResponseEntity.status(200).header("Content-Type", "application/json").body(new AssignmentResponse(assignment)); } diff --git a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java index be8e28f..13570d7 100644 --- a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java @@ -18,25 +18,24 @@ public class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); - private static Scanner inputScanner = null; + public static Scanner inputScanner = null; private static int fileIndex = 0; private static int unretrievableInputLines = 0; - private static int duplicateIdUrlEntries = 0; + public static int duplicateIdUrlEntries = 0; public static int jsonBatchSize = 3000; private static final String utf8Charset = "UTF-8"; public static String inputFileFullPath; - private static String userDir = System.getProperty("user.dir") + File.separator; + private static String workingDir = System.getProperty("user.dir") + File.separator; - public FileUtils() + + public FileUtils() throws RuntimeException { - inputFileFullPath = userDir + "src" + File.separator + "main" + File.separator + "resources"; + inputFileFullPath = workingDir + "src" + File.separator + "main" + File.separator + "resources"; String resourceFileName = "testInputFiles" + File.separator + "orderedList1000.json"; inputFileFullPath += File.separator + resourceFileName; InputStream inputStream = getClass().getClassLoader().getResourceAsStream(resourceFileName); - if ( inputStream == null ) { - logger.error("No resourceFile was found with name \"" + resourceFileName + "\"."); - return; - } + if ( inputStream == null ) + throw new RuntimeException("No resourceFile was found with name \"" + resourceFileName + "\"."); logger.debug("Going to retrieve the data from the inputResourceFile: " + resourceFileName);