lsmyrnaios
/
UrlsController
1
0
Fork 2
Code Issues Pull Requests Releases 18 Wiki Activity

Make the S3-file-uploading operation more resilient to connection-issues, by increasing timeout-periods and implementing a retry-logic upon timeout-error.

This commit is contained in:
Lampros Smyrnaios 2024-11-09 18:05:25 +02:00
parent d1a4f07b0a
commit 69183a2e96
2 changed files with 31 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -246,7 +246,7 @@ public class FileUtils {
String fileFullPath = targetDirectory + fileName; // The fullPath to the local file (which has the previous name).
String s3Url = null;
try {
s3Url = s3ObjectStore.uploadToS3(filenameForS3, fileFullPath, mimeType);
s3Url = s3ObjectStore.uploadToS3(filenameForS3, fileFullPath, mimeType, 0);
} catch (ConnectException ce) {
logger.error("Could not connect with the S3 Object Store! " + ce.getMessage());
throw ce;

35
src/main/java/eu/openaire/urls_controller/util/S3ObjectStore.java
View File

@ -10,6 +10,7 @@ import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -44,6 +45,8 @@ public class S3ObjectStore {
@PostConstruct
public void init() throws Exception {
this.minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).region(region).build();
// Default timeouts are: conn: 30secs, read: 60secs, write: 30secs.
this.minioClient.setTimeout(TimeUnit.MINUTES.toMillis(1), TimeUnit.MINUTES.toMillis(5), TimeUnit.MINUTES.toMillis(5));
boolean bucketExists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
@ -77,17 +80,30 @@ public class S3ObjectStore {
* @param fileObjKeyName = "**File object key name**";
* @param fileFullPath = "**Path of the file to upload**";
* @param contentType
* @param retryCount
* @return the url of the uploaded file
*/
public String uploadToS3(String fileObjKeyName, String fileFullPath, String contentType) throws Exception
public String uploadToS3(String fileObjKeyName, String fileFullPath, String contentType, int retryCount) throws Exception
{
if ( retryCount > 3 ) {
String errorMsg = "Could not upload file with keyName \"" + fileObjKeyName + "\" to S3, after " + (retryCount -1) + " retries!";
logger.error(errorMsg);
return null;
}
if ( contentType == null )
contentType = extractContentTypeFromFileNameKey(fileObjKeyName, fileFullPath);
minioClient.uploadObject(UploadObjectArgs.builder()
.bucket(bucketName)
.object(fileObjKeyName).filename(fileFullPath)
.contentType(contentType).build());
try {
minioClient.uploadObject(UploadObjectArgs.builder().bucket(bucketName).object(fileObjKeyName).filename(fileFullPath).contentType(contentType).build());
} catch (Exception e) {
String errorMsg = e.getMessage();
if ( (errorMsg != null) && errorMsg.toLowerCase().contains("timeout") ) {
sleep30Seconds();
return uploadToS3(fileObjKeyName, fileFullPath, contentType, (++retryCount));
} else
throw e;
}
// What if the fileObjKeyName already exists? (Default action from S3-Object-Store --> overwrite)
// Each Worker handles some of these cases, but in case of id-urls splitting between different workers or re-attempting some temporarily faulty urls later, duplicate fileNames may appear.
@ -198,4 +214,13 @@ public class S3ObjectStore {
return contentType;
}
private void sleep30Seconds() {
try {
Thread.sleep(30_000);
} catch (InterruptedException ie) {
logger.warn("Sleeping was interrupted!");
}
}
}