package org.gcube.contentmanagement.blobstorage.transport.plugin; import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.List; import java.util.Objects; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.waiters.WaiterResponse; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Bucket; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.HeadBucketResponse; import software.amazon.awssdk.services.s3.model.ListBucketsRequest; import software.amazon.awssdk.services.s3.model.ListBucketsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.S3Response; import software.amazon.awssdk.services.s3.waiters.S3Waiter; public class BucketOperator { private static BucketOperator single_instance; private static S3Client client; private static Logger logger = LoggerFactory.getLogger(BucketOperator.class); private BucketOperator(S3Client client) { if(Objects.isNull(client)) throw new RemoteBackendException("S3Client not initialized properly"); this.client=client; } public static BucketOperator getInstance(S3Client client) { if(Objects.isNull(single_instance)) single_instance=new BucketOperator(client); return single_instance; } /** * Creates a bucket in the given region with the name specified. * @param s3 * @param bucketName * @param region */ public static void createBucket(S3Client s3, String bucketName, Region region) { CreateBucketRequest createBucketRequest = CreateBucketRequest.builder() .bucket(bucketName).createBucketConfiguration( CreateBucketConfiguration.builder().locationConstraint(region.id()).build()) .build(); System.out.println(s3.createBucket(createBucketRequest).toString()); } /** * Lists the buckets in your account. * @param s3 */ public void listBuckets(S3Client s3) { ListBucketsRequest req = ListBucketsRequest.builder().build(); ListBucketsResponse res = s3.listBuckets(req); res.buckets().stream().forEach(x->System.out.println(x.name())); } /** * Check if a given bucket exist in your account. * @param s3 */ public static boolean isBucket(S3Client s3, String bucketName) { ListBucketsRequest req = ListBucketsRequest.builder().build(); ListBucketsResponse res = s3.listBuckets(req); for(Bucket bucket:res.buckets()) { if (bucket.name().equals(bucketName)) return true; } return false; } // Create a bucket by using a S3Waiter object public static void createBucketWaiter( S3Client s3Client, String bucketName, Region region) { try { S3Waiter s3Waiter = s3Client.waiter(); CreateBucketRequest bucketRequest = CreateBucketRequest.builder() .bucket(bucketName) .createBucketConfiguration( CreateBucketConfiguration.builder() .locationConstraint(region.id()) .build()) .build(); s3Client.createBucket(bucketRequest); HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder() .bucket(bucketName) .build(); // Wait until the bucket is created and print out the response WaiterResponse waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait); waiterResponse.matched().response().ifPresent(System.out::println); logger.info(bucketName +" is ready"); } catch (S3Exception e) { System.err.println(e.awsErrorDetails().errorMessage()); System.exit(1); } } /** * Before you can delete a S3 bucket, you must ensure that the bucket * is empty or the service will return an error. If you have a versioned * bucket, you must also delete any versioned objects that are in the bucket. * * @param s3 * @param bucketName * @param region */ public static void deleteBucketWithContents(final S3Client s3, final String bucketName, final Region region) { ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(bucketName).build(); ListObjectsV2Response response; do { response = s3.listObjectsV2(request); for(S3Object s3Obj : response.contents()) { s3.deleteObject(DeleteObjectRequest.builder() .bucket(bucketName).key(s3Obj.key()).build()); } request = ListObjectsV2Request.builder().bucket(bucketName) .continuationToken(response.continuationToken()).build(); }while(response.isTruncated()); //Now the bucket is empty, delete the bucket deleteEmptyBucket(s3, bucketName); } /** * Deletes the bucket specified, given the bucket is empty. * @param s3 * @param bucketName */ public static void deleteEmptyBucket(final S3Client s3, final String bucketName) { DeleteBucketRequest req = DeleteBucketRequest.builder().bucket(bucketName).build(); System.out.println(s3.deleteBucket(req).toString()); } /** * Copy an object from a bucket to another. If the destination bucket is the same, the key will be copied in the same bucket. * @param s3 * @param objectKey * @param fromBucket * @param toBucket */ public static String copyObject(final S3Client s3, String sourceKey, String fromBucket, String destKey, String toBucket) { String encodedUrl = null; try { encodedUrl = URLEncoder.encode(fromBucket + "/" + sourceKey, StandardCharsets.UTF_8.toString()); } catch (UnsupportedEncodingException e) { System.out.println("URL could not be encoded: " + e.getMessage()); } CopyObjectResponse response = s3.copyObject(CopyObjectRequest.builder().copySource(encodedUrl).destinationBucket(toBucket).destinationKey(destKey).build()); return response.copyObjectResult().toString(); } /** * * @param bucket * @param key * @param localPath * @return */ public static GetObjectResponse getObject(String bucket, String key, String localPath) { GetObjectRequest getObjectRequest = GetObjectRequest.builder() .bucket(bucket) .key(key) .build(); return client.getObject(getObjectRequest, Paths.get(localPath)); } /** * * @param bucket * @param key * @param file * @param replace * @return */ public static S3Response putObjectOld(String bucket, String key, File file, boolean replace) { if(!replace) { GetObjectRequest getObjectRequest = GetObjectRequest.builder() .bucket(bucket) .key(key) .build(); try { return client.getObject(getObjectRequest).response(); }catch (NoSuchKeyException e) { } } PutObjectRequest objectRequest = PutObjectRequest.builder() .bucket(bucket) .key(key) .build(); return client.putObject(objectRequest, RequestBody.fromFile(file)); } /** * * @param bucket * @param key * @param file * @param replace * @return */ public static String putObject(String bucket, String key, File file, boolean replace) { if(!replace) { GetObjectRequest getObjectRequest = GetObjectRequest.builder() .bucket(bucket) .key(key) .build(); try { client.getObject(getObjectRequest).response(); }catch (NoSuchKeyException e) { } return key; } PutObjectRequest objectRequest = PutObjectRequest.builder() .bucket(bucket) .key(key) .build(); client.putObject(objectRequest, RequestBody.fromFile(file)); return key; } /** * * @param bucket * @param key * @return */ public static String deleteObject(String bucket, String key) { DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() .bucket(bucket) .key(key) .build(); DeleteObjectResponse response=client.deleteObject(deleteObjectRequest); return response.toString(); } /** * Get object length in bytes * @param bucket * @param key * @return * @throws IOException */ public static Long getObjectSize( String bucket, String key) throws IOException { GetObjectRequest getObjectRequest = GetObjectRequest.builder() .bucket(bucket) .key(key) .build(); return client.getObject(getObjectRequest).response().contentLength(); } }