- Change the fileNames' structure in the S3-ObjectStore.

- Update dependencies.
master
Lampros Smyrnaios 2 years ago
parent 48670f3399
commit 5e4fad2479

@ -1,5 +1,5 @@
plugins {
id 'org.springframework.boot' version '2.6.5'
id 'org.springframework.boot' version '2.6.6'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java'
}

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

@ -17,7 +17,7 @@ if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then
justInstall=0
fi
gradleVersion="7.4.1"
gradleVersion="7.4.2"
if [[ justInstall -eq 0 ]]; then

@ -17,7 +17,8 @@ import java.sql.Timestamp;
"size",
"hash",
"location",
"provenance"
"provenance",
"datasourceId"
})
public class Payload {
@ -48,9 +49,13 @@ public class Payload {
@JsonProperty("provenance")
private String provenance; // "crawl:<PluginName>"
@JsonProperty("provenance")
private String datasourceId; // "crawl:<PluginName>"
public Payload() {}
public Payload(String id, String original_url, String actual_url, Timestamp timestamp_acquired, String mime_type, Long size, String hash, String location, String provenance) {
public Payload(String id, String original_url, String actual_url, Timestamp timestamp_acquired, String mime_type, Long size, String hash, String location, String provenance, String datasourceId) {
this.id = id;
this.original_url = original_url;
this.actual_url = actual_url;
@ -60,6 +65,7 @@ public class Payload {
this.hash = hash;
this.location = location;
this.provenance = provenance;
this.datasourceId = datasourceId;
}
public String getId() {
@ -134,18 +140,29 @@ public class Payload {
this.provenance = provenance;
}
public String getDatasourceId() {
return datasourceId;
}
public void setDatasourceId(String datasourceId) {
this.datasourceId = datasourceId;
}
@Override
public String toString() {
return "Payload{" +
"id='" + id + '\'' +
", original_url='" + original_url + '\'' +
", actual_url='" + actual_url + '\'' +
", timestamp_acquired='" + timestamp_acquired + '\'' +
", timestamp_acquired=" + timestamp_acquired +
", mime_type='" + mime_type + '\'' +
", size='" + size + '\'' +
", md5='" + hash + '\'' +
", size=" + size +
", hash='" + hash + '\'' +
", location='" + location + '\'' +
", provenance='" + provenance + '\'' +
", datasourceId='" + datasourceId + '\'' +
'}';
}
}

@ -90,7 +90,13 @@ public class FileUtils {
@Value("services.pdfaggregation.controller.baseTargetLocation")
private String baseTargetLocation;
public static DecimalFormat df = new DecimalFormat("0.00");
private final Pattern FILENAME_WITH_EXTENSION = Pattern.compile(".*/([\\w_:()]+\\.[\\w]{2,10})$");
private static final Pattern FILENAME_WITH_EXTENSION = Pattern.compile(".*/([\\w_:()]+\\.[\\w]{2,10})$");
// The following regex might be usefull in a future scenario. It extracts the "plain-filename" and "file-ID" and the "file-extension". It may even be merged with the above regex.
// Possible full-filenames are: "ID.pdf", "ID(12).pdf"
//private static final Pattern FILENAME_ID_EXTENSION = Pattern.compile("(([^.()]+)[^.]*)(\.[\w]{2,10})$");
private final int numOfFullTextsPerBatch = 70; // The HTTP-headers cannot be too large (It failed with 100 fileNames).
public UploadFullTextsResponse getAndUploadFullTexts(List<UrlReport> urlReports, HttpServletRequest request, long assignmentsBatchCounter, String workerId) {
@ -243,6 +249,15 @@ public class FileUtils {
// Let's try to upload the file to S3 and update the payloads, either in successful file-uploads (right-away) or not (in the end).
try {
// Prepare the filename as: "datasourceid/publicationid(123)::hash.pdf"
// All related payloads point to this exact same file and have the same datasourceId.
Payload firstRelatedPayload = fileRelatedPayloads.stream().findFirst().get();
String datasourceId = firstRelatedPayload.getDatasourceId();
String hash = firstRelatedPayload.getHash();
String[] fileNameData = fileName.split("\\.");
fileName = datasourceId + "/" + fileNameData[0] + "::" + hash + "." + fileNameData[1];
String s3Url = s3ObjectStore.uploadToS3(fileName, fileFullPath);
setFullTextForMultiplePayloads(fileRelatedPayloads, s3Url);
numUploadedFiles ++;

Loading…
Cancel
Save