301 lines
10 KiB
Java
301 lines
10 KiB
Java
package org.gcube.contentmanagement.blobstorage.transport.plugin;
|
|
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.io.UnsupportedEncodingException;
|
|
import java.net.URLEncoder;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.nio.file.Paths;
|
|
import java.util.List;
|
|
import java.util.Objects;
|
|
|
|
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import software.amazon.awssdk.core.sync.RequestBody;
|
|
import software.amazon.awssdk.core.waiters.WaiterResponse;
|
|
import software.amazon.awssdk.regions.Region;
|
|
import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
|
|
import software.amazon.awssdk.services.s3.S3Client;
|
|
import software.amazon.awssdk.services.s3.model.Bucket;
|
|
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
|
|
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
|
|
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
|
|
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
|
|
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
|
|
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
|
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
|
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
|
|
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
|
|
import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
|
|
import software.amazon.awssdk.services.s3.model.ListBucketsRequest;
|
|
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
|
|
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
|
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
|
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
|
|
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
|
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
|
|
import software.amazon.awssdk.services.s3.model.S3Exception;
|
|
import software.amazon.awssdk.services.s3.model.S3Object;
|
|
import software.amazon.awssdk.services.s3.model.S3Response;
|
|
import software.amazon.awssdk.services.s3.waiters.S3Waiter;
|
|
|
|
|
|
public class BucketOperator {
|
|
|
|
private static BucketOperator single_instance;
|
|
private static S3Client client;
|
|
private static Logger logger = LoggerFactory.getLogger(BucketOperator.class);
|
|
|
|
private BucketOperator(S3Client client) {
|
|
if(Objects.isNull(client))
|
|
throw new RemoteBackendException("S3Client not initialized properly");
|
|
this.client=client;
|
|
}
|
|
|
|
public static BucketOperator getInstance(S3Client client) {
|
|
if(Objects.isNull(single_instance))
|
|
single_instance=new BucketOperator(client);
|
|
return single_instance;
|
|
}
|
|
|
|
/**
|
|
* Creates a bucket in the given region with the name specified.
|
|
* @param s3
|
|
* @param bucketName
|
|
* @param region
|
|
*/
|
|
public static void createBucket(S3Client s3, String bucketName, Region region) {
|
|
CreateBucketRequest createBucketRequest = CreateBucketRequest.builder()
|
|
.bucket(bucketName).createBucketConfiguration(
|
|
CreateBucketConfiguration.builder().locationConstraint(region.id()).build())
|
|
.build();
|
|
System.out.println(s3.createBucket(createBucketRequest).toString());
|
|
}
|
|
|
|
/**
|
|
* Lists the buckets in your account.
|
|
* @param s3
|
|
*/
|
|
public void listBuckets(S3Client s3) {
|
|
ListBucketsRequest req = ListBucketsRequest.builder().build();
|
|
ListBucketsResponse res = s3.listBuckets(req);
|
|
res.buckets().stream().forEach(x->System.out.println(x.name()));
|
|
}
|
|
|
|
|
|
/**
|
|
* Check if a given bucket exist in your account.
|
|
* @param s3
|
|
*/
|
|
public static boolean isBucket(S3Client s3, String bucketName) {
|
|
ListBucketsRequest req = ListBucketsRequest.builder().build();
|
|
ListBucketsResponse res = s3.listBuckets(req);
|
|
for(Bucket bucket:res.buckets()) {
|
|
if (bucket.name().equals(bucketName))
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Create a bucket by using a S3Waiter object
|
|
public static void createBucketWaiter( S3Client s3Client, String bucketName, Region region) {
|
|
|
|
try {
|
|
S3Waiter s3Waiter = s3Client.waiter();
|
|
CreateBucketRequest bucketRequest = CreateBucketRequest.builder()
|
|
.bucket(bucketName)
|
|
.createBucketConfiguration(
|
|
CreateBucketConfiguration.builder()
|
|
.locationConstraint(region.id())
|
|
.build())
|
|
.build();
|
|
|
|
s3Client.createBucket(bucketRequest);
|
|
HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder()
|
|
.bucket(bucketName)
|
|
.build();
|
|
|
|
// Wait until the bucket is created and print out the response
|
|
WaiterResponse<HeadBucketResponse> waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait);
|
|
waiterResponse.matched().response().ifPresent(System.out::println);
|
|
logger.info(bucketName +" is ready");
|
|
|
|
} catch (S3Exception e) {
|
|
System.err.println(e.awsErrorDetails().errorMessage());
|
|
System.exit(1);
|
|
}
|
|
}
|
|
/**
|
|
* Before you can delete a S3 bucket, you must ensure that the bucket
|
|
* is empty or the service will return an error. If you have a versioned
|
|
* bucket, you must also delete any versioned objects that are in the bucket.
|
|
*
|
|
* @param s3
|
|
|
|
* @param bucketName
|
|
* @param region
|
|
*/
|
|
|
|
|
|
public static void deleteBucketWithContents(final S3Client s3, final String bucketName, final Region region) {
|
|
|
|
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(bucketName).build();
|
|
ListObjectsV2Response response;
|
|
do {
|
|
response = s3.listObjectsV2(request);
|
|
for(S3Object s3Obj : response.contents()) {
|
|
s3.deleteObject(DeleteObjectRequest.builder()
|
|
.bucket(bucketName).key(s3Obj.key()).build());
|
|
}
|
|
request = ListObjectsV2Request.builder().bucket(bucketName) .continuationToken(response.continuationToken()).build();
|
|
}while(response.isTruncated());
|
|
|
|
|
|
|
|
//Now the bucket is empty, delete the bucket
|
|
deleteEmptyBucket(s3, bucketName);
|
|
}
|
|
|
|
/**
|
|
* Deletes the bucket specified, given the bucket is empty.
|
|
* @param s3
|
|
* @param bucketName
|
|
*/
|
|
public static void deleteEmptyBucket(final S3Client s3, final String bucketName) {
|
|
|
|
DeleteBucketRequest req = DeleteBucketRequest.builder().bucket(bucketName).build();
|
|
System.out.println(s3.deleteBucket(req).toString());
|
|
}
|
|
|
|
/**
|
|
* Copy an object from a bucket to another. If the destination bucket is the same, the key will be copied in the same bucket.
|
|
* @param s3
|
|
* @param objectKey
|
|
* @param fromBucket
|
|
* @param toBucket
|
|
*/
|
|
public static String copyObject(final S3Client s3, String sourceKey, String fromBucket, String destKey, String toBucket) {
|
|
String encodedUrl = null;
|
|
try {
|
|
encodedUrl = URLEncoder.encode(fromBucket + "/" + sourceKey, StandardCharsets.UTF_8.toString());
|
|
} catch (UnsupportedEncodingException e) {
|
|
System.out.println("URL could not be encoded: " + e.getMessage());
|
|
}
|
|
CopyObjectResponse response = s3.copyObject(CopyObjectRequest.builder().copySource(encodedUrl).destinationBucket(toBucket).destinationKey(destKey).build());
|
|
return response.copyObjectResult().toString();
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param bucket
|
|
* @param key
|
|
* @param localPath
|
|
* @return
|
|
*/
|
|
public static GetObjectResponse getObject(String bucket, String key, String localPath) {
|
|
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
|
|
.bucket(bucket)
|
|
.key(key)
|
|
.build();
|
|
|
|
return client.getObject(getObjectRequest, Paths.get(localPath));
|
|
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param bucket
|
|
* @param key
|
|
* @param file
|
|
* @param replace
|
|
* @return
|
|
*/
|
|
public static S3Response putObjectOld(String bucket, String key, File file, boolean replace) {
|
|
if(!replace) {
|
|
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
|
|
.bucket(bucket)
|
|
.key(key)
|
|
.build();
|
|
try {
|
|
return client.getObject(getObjectRequest).response();
|
|
}catch (NoSuchKeyException e) {
|
|
|
|
}
|
|
}
|
|
PutObjectRequest objectRequest = PutObjectRequest.builder()
|
|
.bucket(bucket)
|
|
.key(key)
|
|
.build();
|
|
|
|
return client.putObject(objectRequest, RequestBody.fromFile(file));
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param bucket
|
|
* @param key
|
|
* @param file
|
|
* @param replace
|
|
* @return
|
|
*/
|
|
public static String putObject(String bucket, String key, File file, boolean replace) {
|
|
if(!replace) {
|
|
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
|
|
.bucket(bucket)
|
|
.key(key)
|
|
.build();
|
|
try {
|
|
client.getObject(getObjectRequest).response();
|
|
|
|
}catch (NoSuchKeyException e) {
|
|
|
|
}
|
|
return key;
|
|
}
|
|
PutObjectRequest objectRequest = PutObjectRequest.builder()
|
|
.bucket(bucket)
|
|
.key(key)
|
|
.build();
|
|
|
|
client.putObject(objectRequest, RequestBody.fromFile(file));
|
|
return key;
|
|
}
|
|
|
|
|
|
/**
|
|
*
|
|
* @param bucket
|
|
* @param key
|
|
* @return
|
|
*/
|
|
public static String deleteObject(String bucket, String key) {
|
|
DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
|
|
.bucket(bucket)
|
|
.key(key)
|
|
.build();
|
|
DeleteObjectResponse response=client.deleteObject(deleteObjectRequest);
|
|
return response.toString();
|
|
}
|
|
|
|
/**
|
|
* Get object length in bytes
|
|
* @param bucket
|
|
* @param key
|
|
* @return
|
|
* @throws IOException
|
|
*/
|
|
|
|
public static Long getObjectSize( String bucket, String key)
|
|
throws IOException {
|
|
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
|
|
.bucket(bucket)
|
|
.key(key)
|
|
.build();
|
|
return client.getObject(getObjectRequest).response().contentLength();
|
|
}
|
|
|
|
|
|
}
|