diff --git a/pom.xml b/pom.xml index ed32d75..7f03e4e 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ - + diff --git a/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/BucketOperator.java b/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/BucketOperator.java index 6f928bd..cd7f17f 100644 --- a/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/BucketOperator.java +++ b/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/BucketOperator.java @@ -6,6 +6,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.util.List; import java.util.Objects; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; @@ -13,9 +14,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.core.waiters.WaiterResponse; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Bucket; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; @@ -24,6 +27,8 @@ import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; import software.amazon.awssdk.services.s3.model.ListBucketsRequest; import software.amazon.awssdk.services.s3.model.ListBucketsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; @@ -31,14 +36,17 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.S3Response; +import software.amazon.awssdk.services.s3.waiters.S3Waiter; + public class BucketOperator { private static BucketOperator single_instance; private static S3Client client; - private Logger logger = LoggerFactory.getLogger(BucketOperator.class); + private static Logger logger = LoggerFactory.getLogger(BucketOperator.class); private BucketOperator(S3Client client) { if(Objects.isNull(client)) @@ -70,12 +78,55 @@ public class BucketOperator { * Lists the buckets in your account. * @param s3 */ - public static void listBuckets(S3Client s3) { + public void listBuckets(S3Client s3) { ListBucketsRequest req = ListBucketsRequest.builder().build(); ListBucketsResponse res = s3.listBuckets(req); res.buckets().stream().forEach(x->System.out.println(x.name())); } + + + /** + * Check if a given bucket exist in your account. + * @param s3 + */ + public static boolean isBucket(S3Client s3, String bucketName) { + ListBucketsRequest req = ListBucketsRequest.builder().build(); + ListBucketsResponse res = s3.listBuckets(req); + for(Bucket bucket:res.buckets()) { + if (bucket.name().equals(bucketName)) + return true; + } + return false; + } + // Create a bucket by using a S3Waiter object + public static void createBucketWaiter( S3Client s3Client, String bucketName, Region region) { + + try { + S3Waiter s3Waiter = s3Client.waiter(); + CreateBucketRequest bucketRequest = CreateBucketRequest.builder() + .bucket(bucketName) + .createBucketConfiguration( + CreateBucketConfiguration.builder() + .locationConstraint(region.id()) + .build()) + .build(); + + s3Client.createBucket(bucketRequest); + HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder() + .bucket(bucketName) + .build(); + + // Wait until the bucket is created and print out the response + WaiterResponse waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait); + waiterResponse.matched().response().ifPresent(System.out::println); + logger.info(bucketName +" is ready"); + + } catch (S3Exception e) { + System.err.println(e.awsErrorDetails().errorMessage()); + System.exit(1); + } + } /** * Before you can delete a S3 bucket, you must ensure that the bucket * is empty or the service will return an error. If you have a versioned @@ -177,10 +228,10 @@ public class BucketOperator { .bucket(bucket) .key(key) .build(); -// client.putObject(objectRequest, RequestBody.fromByteBuffer(getRandomByteBuffer(10_000))); + return client.putObject(objectRequest, RequestBody.fromFile(file)); - - } + } + /** diff --git a/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/S3PluginManager.java b/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/S3PluginManager.java index d39bd4f..3b1a104 100644 --- a/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/S3PluginManager.java +++ b/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/S3PluginManager.java @@ -6,7 +6,7 @@ import java.io.IOException; import java.net.UnknownHostException; import java.util.List; import java.util.Map; -import java.util.Objects; + import org.gcube.contentmanagement.blobstorage.resource.MemoryType; import org.gcube.contentmanagement.blobstorage.resource.MyFile; @@ -32,10 +32,9 @@ import java.net.URI; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; + import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; - - import com.mongodb.MongoException; public class S3PluginManager extends TransportManager { @@ -47,9 +46,9 @@ public class S3PluginManager extends TransportManager { // THE FOLLOWING FIELDS SHOULD BE RETRIEVED FROM SERVICEENDPOINT static final String ACCESS_KEY="UXMKEBMZBDROS5K9HREG"; static final String SECRET_ACCESS_KEY="0Mqu8gmzcZ09wuqqKZwNmuS5bYC549MlZhvfgv6W"; - Region region = Region.US_EAST_1; - final String END_POINT = "https://s3.wasabisys.com"; - static final String bucket="gcube-home-test-access"; + Region region = Region.EU_NORTH_1; +// final String END_POINT = "https://s3.wasabisys.com"; + static String bucket="g3-test1data"; // gcube-test;//"gcube-home-test-access"; @Override @@ -69,14 +68,52 @@ public class S3PluginManager extends TransportManager { } @Override - public void initBackend(String[] server, String user, String pass, MemoryType memoryType, String[] dbNames, + public void initBackend(String[] server, String accessKey, String secretAccessKey, MemoryType memoryType, String[] dbNames, String writeConcern, String readConcern) { - AwsSessionCredentials awsCreds = AwsSessionCredentials.create(ACCESS_KEY, SECRET_ACCESS_KEY, ""); +// AwsSessionCredentials awsCreds = AwsSessionCredentials.create(ACCESS_KEY, SECRET_ACCESS_KEY, ""); + AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretAccessKey, ""); +// s3 = S3Client.builder().credentialsProvider( +// StaticCredentialsProvider.create(awsCreds)) +// .endpointOverride(URI.create(END_POINT)).region(region).build(); +// region=Region.of(dbNames[0]); +// region=Region.of("The Netherlands"); +// logger.debug("connection to region: "+region); + region = Region.of("eu-nl"); + s3 = S3Client.builder().credentialsProvider( + StaticCredentialsProvider.create(awsCreds)) + .endpointOverride(URI.create(server[0])).region(region).build(); +// without region specification +// s3 = S3Client.builder().credentialsProvider( +// StaticCredentialsProvider.create(awsCreds)) +// .endpointOverride(URI.create(server[0])).build(); + + + } + + +// @Override + public void initBackendzad(String[] server, String accessKey, String secretAccessKey, MemoryType memoryType, String[] dbNames, + String writeConcern, String readConcern) { + region = Region.of("IT"); +// AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretAccessKey, ""); + AwsSessionCredentials awsCreds = AwsSessionCredentials.create("1168063577b54b30bd35e9ed209e698e", "e60be85b39dc4e1e816f6d922c1f2c3a", "gAAAAABf3MQXCVgepxjLLPAaPb063qABbuMVERNoW7CrQgQpu-Phea-YG1aQEhBwbnQZqkAZMUfI3DaMbFdCPZ7sTJn2RY3hwP8y1qBQmt5oRTxJAHO01JdLcTAOtBLH7et1KPdo-AOLvuwWclYYI2nvztqeT2FhNK5irTY004tm_1pgwvuHRfE"); +// s3 = S3Client.builder().region(newRegion) +// .credentialsProvider(StaticCredentialsProvider.create(awsCreds)) +// .httpClientBuilder(ApacheHttpClient.builder().connectionTimeout(Duration.ofSeconds(30)) +// .connectionMaxIdleTime(Duration.ofSeconds(30)).socketTimeout(Duration.ofSeconds(60))); +// s3 = s3.endpointOverride(new URI("http://xxxxxxx")); + + s3 = S3Client.builder().credentialsProvider( - StaticCredentialsProvider.create(awsCreds)) - .endpointOverride(URI.create(END_POINT)).region(region).build(); + StaticCredentialsProvider.create(awsCreds)) + .endpointOverride(URI.create("https://vsa-0000000e-it-momit-01.zadarazios.com:443")).region(region).build(); + + + } + + @Override public Object get(Download download) throws FileNotFoundException, IOException { @@ -85,10 +122,13 @@ public class S3PluginManager extends TransportManager { @Override public String put(Upload upload) throws FileNotFoundException, IOException { - logger.info("put method invoked "); + logger.trace("put method invoked "); + String bucketName=Utils.convertToS3Format(upload.getResource().getRootPath()); + if (!BucketOperator.isBucket(s3, bucketName)) + BucketOperator.createBucketWaiter(s3, bucketName, region); File file= new File(upload.getResource().getLocalPath()); // return BucketOperator.getInstance(s3).putObject(bucket, Utils.convertToS3Format(upload.getResource().getRemotePath()), file).toString(); - int hashcode=BucketOperator.getInstance(s3).putObject(bucket, Utils.convertToS3Format(upload.getResource().getRemotePath()), file, upload.isReplaceOption()).hashCode(); + int hashcode=BucketOperator.getInstance(s3).putObject(bucketName, Utils.convertToS3Format(upload.getResource().getRemotePath()), file, upload.isReplaceOption()).hashCode(); return hashcode+""; } @@ -113,7 +153,7 @@ public class S3PluginManager extends TransportManager { public long getSize(String key, MyFile file) { logger.info("remoteObject identifier "+key); try { - return BucketOperator.getObjectSize(bucket, Utils.convertToS3Format(file.getRemotePath())); + return BucketOperator.getObjectSize(Utils.convertToS3Format(file.getRootPath()), Utils.convertToS3Format(file.getRemotePath())); } catch (IOException e) { e.printStackTrace(); throw new RemoteBackendException("Problem during getSize Operation on s3 cloud storage"); @@ -231,5 +271,6 @@ public class S3PluginManager extends TransportManager { public String duplicateFile(DuplicateFile duplicate) { throw new RemoteBackendException("method not implemented yet on s3 plugin"); } + } diff --git a/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/Utils/Utils.java b/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/Utils/Utils.java index 8c6b692..51526d5 100644 --- a/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/Utils/Utils.java +++ b/src/main/java/org/gcube/contentmanagement/blobstorage/transport/plugin/Utils/Utils.java @@ -8,6 +8,10 @@ public class Utils { if( objectName.substring(objectName.length()-1).contains(Costants.FILE_SEPARATOR)) { objectName=objectName.substring(0, objectName.length()-1); } + if( objectName.substring(0).contains(Costants.FILE_SEPARATOR)) { + objectName=objectName.substring(1, objectName.length()); + } + objectName=objectName.toLowerCase(); return objectName.replaceAll(Costants.FILE_SEPARATOR, "-"); }