- Show the percentage of the "NumFullTextsFound", in the logs.

- Update dependencies.
This commit is contained in:
Lampros Smyrnaios 2022-03-28 14:29:31 +03:00
parent e587b2ca6c
commit 48670f3399
4 changed files with 10 additions and 8 deletions

View File

@ -1,5 +1,5 @@
plugins { plugins {
id 'org.springframework.boot' version '2.6.4' id 'org.springframework.boot' version '2.6.5'
id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java' id 'java'
} }
@ -40,12 +40,12 @@ dependencies {
implementation "org.projectlombok:lombok:1.18.22" implementation "org.projectlombok:lombok:1.18.22"
// https://mvnrepository.com/artifact/com.google.guava/guava // https://mvnrepository.com/artifact/com.google.guava/guava
implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' implementation group: 'com.google.guava', name: 'guava', version: '31.1-jre'
// https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
implementation 'io.minio:minio:8.3.6' implementation 'io.minio:minio:8.3.7'
// https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp // https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp
implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3'

View File

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

View File

@ -17,7 +17,7 @@ if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then
justInstall=0 justInstall=0
fi fi
gradleVersion="7.4" gradleVersion="7.4.1"
if [[ justInstall -eq 0 ]]; then if [[ justInstall -eq 0 ]]; then

View File

@ -21,6 +21,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.sql.Types; import java.sql.Types;
import java.text.DecimalFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -88,7 +89,7 @@ public class FileUtils {
@Value("services.pdfaggregation.controller.baseTargetLocation") @Value("services.pdfaggregation.controller.baseTargetLocation")
private String baseTargetLocation; private String baseTargetLocation;
public static DecimalFormat df = new DecimalFormat("0.00");
private final Pattern FILENAME_WITH_EXTENSION = Pattern.compile(".*/([\\w_:()]+\\.[\\w]{2,10})$"); private final Pattern FILENAME_WITH_EXTENSION = Pattern.compile(".*/([\\w_:()]+\\.[\\w]{2,10})$");
private final int numOfFullTextsPerBatch = 70; // The HTTP-headers cannot be too large (It failed with 100 fileNames). private final int numOfFullTextsPerBatch = 70; // The HTTP-headers cannot be too large (It failed with 100 fileNames).
@ -107,7 +108,8 @@ public class FileUtils {
// Get the file-locations. // Get the file-locations.
int numFullTextsFound = 0; int numFullTextsFound = 0;
int numFilesFoundFromPreviousAssignmentsBatches = 0; int numFilesFoundFromPreviousAssignmentsBatches = 0;
HashMultimap<String, Payload> allFileNamesWithPayloads = HashMultimap.create((urlReports.size() / 5), 3); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it. int urlReportsSize = urlReports.size();
HashMultimap<String, Payload> allFileNamesWithPayloads = HashMultimap.create((urlReportsSize / 5), 3); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it.
String getFileLocationForHashQuery = "select `location` from " + databaseName + ".payload where `hash` = ? limit 1" ; String getFileLocationForHashQuery = "select `location` from " + databaseName + ".payload where `hash` = ? limit 1" ;
final int[] hashArgType = new int[] {Types.VARCHAR}; final int[] hashArgType = new int[] {Types.VARCHAR};
@ -167,7 +169,7 @@ public class FileUtils {
ImpalaConnector.databaseLock.unlock(); // The remaining work of this function does not use the database. ImpalaConnector.databaseLock.unlock(); // The remaining work of this function does not use the database.
logger.info("NumFullTextsFound by assignments_" + assignmentsBatchCounter + " = " + numFullTextsFound + " (out of " + urlReports.size() + ")."); logger.info("NumFullTextsFound by assignments_" + assignmentsBatchCounter + " = " + numFullTextsFound + " (out of " + urlReportsSize + " | about " + df.format(numFullTextsFound * 100.0 / urlReportsSize) + "%).");
logger.debug("NumFilesFoundFromPreviousAssignmentsBatches = " + numFilesFoundFromPreviousAssignmentsBatches); logger.debug("NumFilesFoundFromPreviousAssignmentsBatches = " + numFilesFoundFromPreviousAssignmentsBatches);
ArrayList<String> allFileNames = new ArrayList<>(allFileNamesWithPayloads.keySet()); ArrayList<String> allFileNames = new ArrayList<>(allFileNamesWithPayloads.keySet());