update to version 0.0.1-SNAPSHOT
parent
4ec8138a06
commit
02ec9b6926
@ -0,0 +1,5 @@
|
||||
# Changelog for storage-manager-s3-plugin
|
||||
|
||||
## [v0.0.1-SNAPSHOT] 2020-11-06
|
||||
* first release
|
||||
|
@ -0,0 +1,57 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.tools</groupId>
|
||||
<artifactId>maven-parent</artifactId>
|
||||
<version>1.1.0</version>
|
||||
</parent>
|
||||
<groupId>org.gcube.contentmanagement</groupId>
|
||||
<artifactId>storage-manager-s3-plugin</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<description>A plugin for storage-manager-core library</description>
|
||||
<properties>
|
||||
<aws.java.sdk.version>2.13.7</aws.java.sdk.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>bom</artifactId>
|
||||
<version>2.15.21</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.contentmanagement</groupId>
|
||||
<artifactId>storage-manager-wrapper</artifactId>
|
||||
<version>[2.5.3-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.contentmanagement</groupId>
|
||||
<artifactId>storage-manager-core</artifactId>
|
||||
<version>[2.9.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>dynamodb</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-lambda-java-core</artifactId>
|
||||
<version>1.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>aws-sdk-java</artifactId>
|
||||
<version>2.13.7</version>
|
||||
</dependency>
|
||||
<!-- <dependency> -->
|
||||
<!-- <groupId>com.instantor.amazon</groupId> -->
|
||||
<!-- <artifactId>S3Client</artifactId> -->
|
||||
<!-- <version>1.0.2</version> -->
|
||||
<!-- </dependency> -->
|
||||
</dependencies>
|
||||
</project>
|
@ -0,0 +1,161 @@
|
||||
package org.gcube.contentmanagement.blobstorage.transport.plugin;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
|
||||
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.ListBucketsRequest;
|
||||
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
|
||||
public class BucketOperator {
|
||||
|
||||
private static BucketOperator single_instance;
|
||||
private static S3Client client;
|
||||
|
||||
private BucketOperator(S3Client client) {
|
||||
if(Objects.isNull(client))
|
||||
throw new RemoteBackendException("S3Client not initialized properly");
|
||||
this.client=client;
|
||||
}
|
||||
|
||||
public static BucketOperator getInstance(S3Client client) {
|
||||
if(Objects.isNull(single_instance))
|
||||
single_instance=new BucketOperator(client);
|
||||
return single_instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a bucket in the given region with the name specified.
|
||||
* @param s3
|
||||
* @param bucketName
|
||||
* @param region
|
||||
*/
|
||||
public static void createBucket(S3Client s3, String bucketName, Region region) {
|
||||
CreateBucketRequest createBucketRequest = CreateBucketRequest.builder()
|
||||
.bucket(bucketName).createBucketConfiguration(
|
||||
CreateBucketConfiguration.builder().locationConstraint(region.id()).build())
|
||||
.build();
|
||||
System.out.println(s3.createBucket(createBucketRequest).toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists the buckets in your account.
|
||||
* @param s3
|
||||
*/
|
||||
public static void listBuckets(S3Client s3) {
|
||||
ListBucketsRequest req = ListBucketsRequest.builder().build();
|
||||
ListBucketsResponse res = s3.listBuckets(req);
|
||||
res.buckets().stream().forEach(x->System.out.println(x.name()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Before you can delete an Amazon S3 bucket, you must ensure that the bucket
|
||||
* is empty or the service will return an error. If you have a versioned
|
||||
* bucket, you must also delete any versioned objects that are in the bucket.
|
||||
*
|
||||
* @param s3
|
||||
|
||||
* @param bucketName
|
||||
* @param region
|
||||
*/
|
||||
|
||||
|
||||
public static void deleteBucketWithContents(final S3Client s3, final String bucketName, final Region region) {
|
||||
|
||||
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(bucketName).build();
|
||||
ListObjectsV2Response response;
|
||||
do {
|
||||
response = s3.listObjectsV2(request);
|
||||
for(S3Object s3Obj : response.contents()) {
|
||||
s3.deleteObject(DeleteObjectRequest.builder()
|
||||
.bucket(bucketName).key(s3Obj.key()).build());
|
||||
}
|
||||
request = ListObjectsV2Request.builder().bucket(bucketName) .continuationToken(response.continuationToken()).build();
|
||||
}while(response.isTruncated());
|
||||
|
||||
|
||||
|
||||
//Now the bucket is empty, delete the bucket
|
||||
deleteEmptyBucket(s3, bucketName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the bucket specified, given the bucket is empty.
|
||||
* @param s3
|
||||
* @param bucketName
|
||||
*/
|
||||
public static void deleteEmptyBucket(final S3Client s3, final String bucketName) {
|
||||
|
||||
DeleteBucketRequest req = DeleteBucketRequest.builder().bucket(bucketName).build();
|
||||
System.out.println(s3.deleteBucket(req).toString());
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param bucket
|
||||
* @param key
|
||||
* @param localPath
|
||||
* @return
|
||||
*/
|
||||
public static GetObjectResponse getObject(String bucket, String key, String localPath) {
|
||||
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.build();
|
||||
|
||||
return client.getObject(getObjectRequest, Paths.get(localPath));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param bucket
|
||||
* @param key
|
||||
* @param file
|
||||
* @return
|
||||
*/
|
||||
public static PutObjectResponse putObject(String bucket, String key, File file) {
|
||||
PutObjectRequest objectRequest = PutObjectRequest.builder()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.build();
|
||||
// client.putObject(objectRequest, RequestBody.fromByteBuffer(getRandomByteBuffer(10_000)));
|
||||
return client.putObject(objectRequest, RequestBody.fromFile(file));
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @param bucket
|
||||
* @param key
|
||||
* @return
|
||||
*/
|
||||
public static String deleteObject(String bucket, String key) {
|
||||
DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.build();
|
||||
DeleteObjectResponse response=client.deleteObject(deleteObjectRequest);
|
||||
return response.toString();
|
||||
}
|
||||
|
||||
|
||||
}
|
@ -0,0 +1,224 @@
|
||||
package org.gcube.contentmanagement.blobstorage.transport.plugin;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.gcube.contentmanagement.blobstorage.resource.MemoryType;
|
||||
import org.gcube.contentmanagement.blobstorage.resource.MyFile;
|
||||
import org.gcube.contentmanagement.blobstorage.resource.StorageObject;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.Copy;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.CopyDir;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.Download;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.DuplicateFile;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.Link;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.Lock;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.Move;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.MoveDir;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.Unlock;
|
||||
import org.gcube.contentmanagement.blobstorage.service.operation.Upload;
|
||||
import org.gcube.contentmanagement.blobstorage.transport.TransportManager;
|
||||
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
||||
|
||||
import java.net.URI;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
|
||||
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.ListBucketsRequest;
|
||||
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
|
||||
|
||||
import com.mongodb.MongoException;
|
||||
|
||||
public class S3PluginManager extends TransportManager {
|
||||
|
||||
Region region = Region.US_EAST_1;
|
||||
final String END_POINT = "https://s3.wasabisys.com";
|
||||
S3Client s3;
|
||||
|
||||
// THE FOLLOWING FIELD SHOULD BE RETRIEVED FROM SERVICEENDPOINT
|
||||
static final String ACCESS_KEY="UXMKEBMZBDROS5K9HREG";
|
||||
static final String SECRET_ACCESS_KEY="0Mqu8gmzcZ09wuqqKZwNmuS5bYC549MlZhvfgv6W";
|
||||
static final String bucket="gcube-home-test-access";
|
||||
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "S3PluginManager";
|
||||
}
|
||||
|
||||
public S3PluginManager(String[] server, String user, String pass, MemoryType memoryType, String[] dbNames,
|
||||
String writeConcern, String readConcern) {
|
||||
initBackend(server,user,pass, memoryType,dbNames, writeConcern, readConcern);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initBackend(String[] server, String user, String pass, MemoryType memoryType, String[] dbNames,
|
||||
String writeConcern, String readConcern) {
|
||||
AwsSessionCredentials awsCreds = AwsSessionCredentials.create(ACCESS_KEY, SECRET_ACCESS_KEY, "");
|
||||
|
||||
s3 = S3Client.builder().credentialsProvider(
|
||||
StaticCredentialsProvider.create(awsCreds))
|
||||
.endpointOverride(URI.create(END_POINT)).region(region).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object get(Download download) throws FileNotFoundException, IOException {
|
||||
return BucketOperator.getInstance(s3).getObject(bucket, download.getRemotePath(), download.getLocalPath());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String put(Upload upload) throws FileNotFoundException, IOException {
|
||||
return BucketOperator.getInstance(s3).getObject(bucket, upload.getRemotePath(), upload.getLocalPath()).toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, StorageObject> getValues(MyFile resource, String bucket, Class<? extends Object> type) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeRemoteFile(String bucket, MyFile resource) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeDir(String remoteDir, MyFile myFile) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSize(String bucket) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String lock(Lock lock) throws Exception {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String unlock(Unlock unlock) throws FileNotFoundException, UnknownHostException, MongoException, Exception {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTTL(String pathServer) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long renewTTL(MyFile resource) throws UnknownHostException, IllegalAccessException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String link(Link link) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String copy(Copy copy) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String move(Move move) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> copyDir(CopyDir copy) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> moveDir(MoveDir move) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileProperty(String remotePath, String property) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getFolderTotalItems(String folderPath) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getFolderTotalVolume(String folderPath) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUserTotalVolume(String user) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUserTotalItems(String user) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isValidId(String id) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(String remoteIdentifier, boolean forceCreation) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getField(String remoteIdentifier, String fieldName) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
s3.close();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFileProperty(String remotePath, String propertyField, String propertyValue) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRemotePath(String bucket) throws UnknownHostException {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exist(String bucket) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String duplicateFile(DuplicateFile duplicate) {
|
||||
throw new RemoteBackendException("method not implemented yet on s3 plugin");
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in New Issue