146 lines
6.3 KiB
Java
146 lines
6.3 KiB
Java
package eu.openaire.urls_controller.util;
|
|
|
|
import io.minio.*;
|
|
import io.minio.messages.Bucket;
|
|
import io.minio.messages.Item;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import javax.annotation.PostConstruct;
|
|
import java.util.List;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
|
|
@Component
|
|
public class S3ObjectStore {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(S3ObjectStore.class);
|
|
|
|
private String s3Protocol = "s3://";
|
|
@Value("${services.pdfaggregation.controller.s3.endpoint}")
|
|
private String endpoint = null; // This is useful to be "public", to test file-locations.
|
|
@Value("${services.pdfaggregation.controller.s3.accessKey}")
|
|
private String accessKey = null;
|
|
@Value("${services.pdfaggregation.controller.s3.secretKey}")
|
|
private String secretKey = null;
|
|
@Value("${services.pdfaggregation.controller.s3.region}")
|
|
private String region = null;
|
|
@Value("${services.pdfaggregation.controller.s3.bucketName}")
|
|
private String bucketName = null;
|
|
|
|
@Value("${services.pdfaggregation.controller.s3.shouldEmptyBucket}")
|
|
private boolean shouldEmptyBucket = false; // Set true only for testing!
|
|
@Value("${services.pdfaggregation.controller.s3.shouldShowAllS3Buckets}")
|
|
private boolean shouldShowAllS3Buckets = false;
|
|
|
|
private MinioClient minioClient;
|
|
|
|
@PostConstruct
|
|
public void init() throws Exception {
|
|
this.minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).region(region).build();
|
|
|
|
boolean bucketExists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
|
|
|
|
// Keep this commented-out to avoid objects-deletion by accident. The code is open-sourced, so it's easy to enable this ability if we really want it (e.g. for testing).
|
|
if ( bucketExists && shouldEmptyBucket ) {
|
|
emptyBucket(bucketName, false);
|
|
//throw new RuntimeException("stop just for test!");
|
|
}
|
|
|
|
// Make the bucket, if not exist.
|
|
if ( !bucketExists ) {
|
|
logger.info("Bucket \"" + bucketName + "\" does not exist! Going to create it..");
|
|
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
|
|
} else
|
|
logger.debug("Bucket \"" + bucketName + "\" already exists.");
|
|
|
|
if ( shouldShowAllS3Buckets ) {
|
|
List<Bucket> buckets = null;
|
|
try {
|
|
buckets = minioClient.listBuckets();
|
|
logger.debug("The buckets in the S3 ObjectStore are:");
|
|
for ( Bucket bucket : buckets ) {
|
|
logger.debug(bucket.name());
|
|
}
|
|
} catch (Exception e) {
|
|
logger.warn("Could not listBuckets: " + e.getMessage());
|
|
}
|
|
}
|
|
}
|
|
|
|
private final Pattern EXTENSION_PATTERN = Pattern.compile("(\\.[^.]+)$");
|
|
|
|
/**
|
|
* @param fileObjKeyName = "**File object key name**";
|
|
* @param fileFullPath = "**Path of the file to upload**";
|
|
* @return the url of the uploaded file
|
|
*/
|
|
public String uploadToS3(String fileObjKeyName, String fileFullPath) throws Exception {
|
|
String contentType = null;
|
|
|
|
// Take the Matcher to retrieve the extension.
|
|
Matcher extensionMatcher = EXTENSION_PATTERN.matcher(fileFullPath);
|
|
if ( extensionMatcher.find() ) {
|
|
String extension = null;
|
|
if ( (extension = extensionMatcher.group(0)) == null )
|
|
contentType = "application/pdf";
|
|
else {
|
|
if ( extension.equals("pdf") )
|
|
contentType = "application/pdf";
|
|
/*else if ( *//* TODO - other-extension-match *//* )
|
|
contentType = "application/EXTENSION"; */
|
|
else
|
|
contentType = "application/pdf"; // Default.
|
|
}
|
|
} else {
|
|
logger.warn("The file with key \"" + fileObjKeyName + "\" does not have a file-extension! Setting the \"pdf\"-mimeType.");
|
|
contentType = "application/pdf";
|
|
}
|
|
|
|
minioClient.uploadObject(UploadObjectArgs.builder()
|
|
.bucket(bucketName)
|
|
.object(fileObjKeyName).filename(fileFullPath)
|
|
.contentType(contentType).build());
|
|
|
|
// TODO - What if the fileObjKeyName already exists? Right now it gets overwritten (unless we add versioning0, which is not currently supported by our S3ObjectStore).
|
|
// Each Worker handles some of these cases, but in case of id-urls splitting between different workers or re-attempting some temporarily faulty urls later,
|
|
// duplicate fileNames may appear and cause file-overwriting from the part of S3ObjectStore.
|
|
|
|
String s3Url = s3Protocol + bucketName + "/" + fileObjKeyName; // Be aware: This url works only if the access to the bucket is public.
|
|
//logger.debug("Uploaded file \"" + fileObjKeyName + "\". The s3Url is: " + s3Url);
|
|
return s3Url;
|
|
}
|
|
|
|
public void emptyBucket(String bucketName, boolean shouldDeleteBucket) throws Exception {
|
|
logger.warn("Going to " + (shouldDeleteBucket ? "delete" : "empty") + " bucket \"" + bucketName + "\"");
|
|
|
|
// First list the objects of the bucket.
|
|
Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName).build());
|
|
|
|
// Then, delete the objects.
|
|
for ( Result<Item> resultItem : results )
|
|
try {
|
|
deleteFile(resultItem.get().objectName(), bucketName);
|
|
} catch (Exception e) {
|
|
logger.warn("Could not remove " + resultItem.get().objectName());
|
|
}
|
|
|
|
if ( shouldDeleteBucket ) {
|
|
// Lastly, delete the empty bucket.
|
|
minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
|
|
}
|
|
|
|
logger.info("Bucket " + bucketName + " was " + (shouldDeleteBucket ? "deleted!" : "emptied!"));
|
|
}
|
|
|
|
public boolean isLocationInStore(String location) {
|
|
return location.startsWith(s3Protocol);
|
|
}
|
|
|
|
private void deleteFile(String fileObjKeyName, String bucketName) throws Exception {
|
|
minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(fileObjKeyName).build());
|
|
}
|
|
}
|