Added isBucket method and createBuckerWaiter method to BucketOperator;

Added temporary methods to S3PluginManager class in order to switch from
different object storages. The method will be deleted when all the
parameters will be retrieved from SE
master
roberto cirillo 3 years ago
parent 9123107e9e
commit 15f16c16d5

@ -46,7 +46,7 @@
<!-- <dependency> -->
<!-- <groupId>software.amazon.awssdk</groupId> -->
<!-- <artifactId>aws-sdk-java</artifactId> -->
<!-- <version>2.13.7</version> -->
<!-- <version>2.5.31</version> -->
<!-- </dependency> -->
<!-- <dependency> -->
<!-- <groupId>com.instantor.amazon</groupId> -->

@ -6,6 +6,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.List;
import java.util.Objects;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
@ -13,9 +14,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.waiters.WaiterResponse;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.Bucket;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
@ -24,6 +27,8 @@ import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
import software.amazon.awssdk.services.s3.model.ListBucketsRequest;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
@ -31,14 +36,17 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.model.S3Response;
import software.amazon.awssdk.services.s3.waiters.S3Waiter;
public class BucketOperator {
private static BucketOperator single_instance;
private static S3Client client;
private Logger logger = LoggerFactory.getLogger(BucketOperator.class);
private static Logger logger = LoggerFactory.getLogger(BucketOperator.class);
private BucketOperator(S3Client client) {
if(Objects.isNull(client))
@ -70,12 +78,55 @@ public class BucketOperator {
* Lists the buckets in your account.
* @param s3
*/
public static void listBuckets(S3Client s3) {
public void listBuckets(S3Client s3) {
ListBucketsRequest req = ListBucketsRequest.builder().build();
ListBucketsResponse res = s3.listBuckets(req);
res.buckets().stream().forEach(x->System.out.println(x.name()));
}
/**
* Check if a given bucket exist in your account.
* @param s3
*/
public static boolean isBucket(S3Client s3, String bucketName) {
ListBucketsRequest req = ListBucketsRequest.builder().build();
ListBucketsResponse res = s3.listBuckets(req);
for(Bucket bucket:res.buckets()) {
if (bucket.name().equals(bucketName))
return true;
}
return false;
}
// Create a bucket by using a S3Waiter object
public static void createBucketWaiter( S3Client s3Client, String bucketName, Region region) {
try {
S3Waiter s3Waiter = s3Client.waiter();
CreateBucketRequest bucketRequest = CreateBucketRequest.builder()
.bucket(bucketName)
.createBucketConfiguration(
CreateBucketConfiguration.builder()
.locationConstraint(region.id())
.build())
.build();
s3Client.createBucket(bucketRequest);
HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder()
.bucket(bucketName)
.build();
// Wait until the bucket is created and print out the response
WaiterResponse<HeadBucketResponse> waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait);
waiterResponse.matched().response().ifPresent(System.out::println);
logger.info(bucketName +" is ready");
} catch (S3Exception e) {
System.err.println(e.awsErrorDetails().errorMessage());
System.exit(1);
}
}
/**
* Before you can delete a S3 bucket, you must ensure that the bucket
* is empty or the service will return an error. If you have a versioned
@ -177,10 +228,10 @@ public class BucketOperator {
.bucket(bucket)
.key(key)
.build();
// client.putObject(objectRequest, RequestBody.fromByteBuffer(getRandomByteBuffer(10_000)));
return client.putObject(objectRequest, RequestBody.fromFile(file));
}
}
/**

@ -6,7 +6,7 @@ import java.io.IOException;
import java.net.UnknownHostException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.gcube.contentmanagement.blobstorage.resource.MemoryType;
import org.gcube.contentmanagement.blobstorage.resource.MyFile;
@ -32,10 +32,9 @@ import java.net.URI;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import com.mongodb.MongoException;
public class S3PluginManager extends TransportManager {
@ -47,9 +46,9 @@ public class S3PluginManager extends TransportManager {
// THE FOLLOWING FIELDS SHOULD BE RETRIEVED FROM SERVICEENDPOINT
static final String ACCESS_KEY="UXMKEBMZBDROS5K9HREG";
static final String SECRET_ACCESS_KEY="0Mqu8gmzcZ09wuqqKZwNmuS5bYC549MlZhvfgv6W";
Region region = Region.US_EAST_1;
final String END_POINT = "https://s3.wasabisys.com";
static final String bucket="gcube-home-test-access";
Region region = Region.EU_NORTH_1;
// final String END_POINT = "https://s3.wasabisys.com";
static String bucket="g3-test1data"; // gcube-test;//"gcube-home-test-access";
@Override
@ -69,14 +68,52 @@ public class S3PluginManager extends TransportManager {
}
@Override
public void initBackend(String[] server, String user, String pass, MemoryType memoryType, String[] dbNames,
public void initBackend(String[] server, String accessKey, String secretAccessKey, MemoryType memoryType, String[] dbNames,
String writeConcern, String readConcern) {
AwsSessionCredentials awsCreds = AwsSessionCredentials.create(ACCESS_KEY, SECRET_ACCESS_KEY, "");
// AwsSessionCredentials awsCreds = AwsSessionCredentials.create(ACCESS_KEY, SECRET_ACCESS_KEY, "");
AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretAccessKey, "");
// s3 = S3Client.builder().credentialsProvider(
// StaticCredentialsProvider.create(awsCreds))
// .endpointOverride(URI.create(END_POINT)).region(region).build();
// region=Region.of(dbNames[0]);
// region=Region.of("The Netherlands");
// logger.debug("connection to region: "+region);
region = Region.of("eu-nl");
s3 = S3Client.builder().credentialsProvider(
StaticCredentialsProvider.create(awsCreds))
.endpointOverride(URI.create(server[0])).region(region).build();
// without region specification
// s3 = S3Client.builder().credentialsProvider(
// StaticCredentialsProvider.create(awsCreds))
// .endpointOverride(URI.create(server[0])).build();
}
// @Override
public void initBackendzad(String[] server, String accessKey, String secretAccessKey, MemoryType memoryType, String[] dbNames,
String writeConcern, String readConcern) {
region = Region.of("IT");
// AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretAccessKey, "");
AwsSessionCredentials awsCreds = AwsSessionCredentials.create("1168063577b54b30bd35e9ed209e698e", "e60be85b39dc4e1e816f6d922c1f2c3a", "gAAAAABf3MQXCVgepxjLLPAaPb063qABbuMVERNoW7CrQgQpu-Phea-YG1aQEhBwbnQZqkAZMUfI3DaMbFdCPZ7sTJn2RY3hwP8y1qBQmt5oRTxJAHO01JdLcTAOtBLH7et1KPdo-AOLvuwWclYYI2nvztqeT2FhNK5irTY004tm_1pgwvuHRfE");
// s3 = S3Client.builder().region(newRegion)
// .credentialsProvider(StaticCredentialsProvider.create(awsCreds))
// .httpClientBuilder(ApacheHttpClient.builder().connectionTimeout(Duration.ofSeconds(30))
// .connectionMaxIdleTime(Duration.ofSeconds(30)).socketTimeout(Duration.ofSeconds(60)));
// s3 = s3.endpointOverride(new URI("http://xxxxxxx"));
s3 = S3Client.builder().credentialsProvider(
StaticCredentialsProvider.create(awsCreds))
.endpointOverride(URI.create(END_POINT)).region(region).build();
StaticCredentialsProvider.create(awsCreds))
.endpointOverride(URI.create("https://vsa-0000000e-it-momit-01.zadarazios.com:443")).region(region).build();
}
@Override
public Object get(Download download) throws FileNotFoundException, IOException {
@ -85,10 +122,13 @@ public class S3PluginManager extends TransportManager {
@Override
public String put(Upload upload) throws FileNotFoundException, IOException {
logger.info("put method invoked ");
logger.trace("put method invoked ");
String bucketName=Utils.convertToS3Format(upload.getResource().getRootPath());
if (!BucketOperator.isBucket(s3, bucketName))
BucketOperator.createBucketWaiter(s3, bucketName, region);
File file= new File(upload.getResource().getLocalPath());
// return BucketOperator.getInstance(s3).putObject(bucket, Utils.convertToS3Format(upload.getResource().getRemotePath()), file).toString();
int hashcode=BucketOperator.getInstance(s3).putObject(bucket, Utils.convertToS3Format(upload.getResource().getRemotePath()), file, upload.isReplaceOption()).hashCode();
int hashcode=BucketOperator.getInstance(s3).putObject(bucketName, Utils.convertToS3Format(upload.getResource().getRemotePath()), file, upload.isReplaceOption()).hashCode();
return hashcode+"";
}
@ -113,7 +153,7 @@ public class S3PluginManager extends TransportManager {
public long getSize(String key, MyFile file) {
logger.info("remoteObject identifier "+key);
try {
return BucketOperator.getObjectSize(bucket, Utils.convertToS3Format(file.getRemotePath()));
return BucketOperator.getObjectSize(Utils.convertToS3Format(file.getRootPath()), Utils.convertToS3Format(file.getRemotePath()));
} catch (IOException e) {
e.printStackTrace();
throw new RemoteBackendException("Problem during getSize Operation on s3 cloud storage");
@ -231,5 +271,6 @@ public class S3PluginManager extends TransportManager {
public String duplicateFile(DuplicateFile duplicate) {
throw new RemoteBackendException("method not implemented yet on s3 plugin");
}
}

@ -8,6 +8,10 @@ public class Utils {
if( objectName.substring(objectName.length()-1).contains(Costants.FILE_SEPARATOR)) {
objectName=objectName.substring(0, objectName.length()-1);
}
if( objectName.substring(0).contains(Costants.FILE_SEPARATOR)) {
objectName=objectName.substring(1, objectName.length());
}
objectName=objectName.toLowerCase();
return objectName.replaceAll(Costants.FILE_SEPARATOR, "-");
}

Loading…
Cancel
Save