From f75f7d86d9531146b689dc2cf9549db84248bd3a Mon Sep 17 00:00:00 2001 From: lucio Date: Fri, 1 Dec 2023 13:04:07 +0100 Subject: [PATCH] jdbc driver updated --- .settings/org.eclipse.jdt.core.prefs | 2 +- .settings/org.eclipse.wst.common.component | 44 ++++++--- docker/storage-settings.properties | 27 ++++-- pom.xml | 24 +++-- .../storage/backend/impl/S3Backend.java | 97 +++++++++---------- .../resources/storage-settings.properties | 12 +-- 6 files changed, 114 insertions(+), 92 deletions(-) diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index e8c450c..29fe717 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -7,5 +7,5 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning -org.eclipse.jdt.core.compiler.release=enabled +org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=11 diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 64553f0..2c9b16b 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -1,5 +1,7 @@ - + + + @@ -29,7 +31,9 @@ - + + + @@ -59,7 +63,9 @@ - + + + @@ -89,7 +95,9 @@ - + + + @@ -119,23 +127,21 @@ - + + + - + uses - - uses - - - uses - - + + + @@ -165,7 +171,9 @@ - + + + @@ -195,7 +203,9 @@ - + + + @@ -225,7 +235,9 @@ - + + + diff --git a/docker/storage-settings.properties b/docker/storage-settings.properties index 144c9f4..85b1c80 100644 --- a/docker/storage-settings.properties +++ b/docker/storage-settings.properties @@ -1,13 +1,22 @@ +#default.bucketName=storagehub-dev +#default.key=SHUBTEST +#default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY +#default.url=http://minio:9000 +#default.createBucket=true +#volatile.bucketName=storagehub-volatile-dev +#volatile.key=SHUBTEST +#volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY +#volatile.url=http://minio:9000 +#volatile.createBucket=true default.bucketName=storagehub-dev -default.key=SHUBTEST -default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY -default.url=http://minio:9000 -default.createBucket=true -volatile.bucketName=storagehub-volatile-dev -volatile.key=SHUBTEST -volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY -volatile.url=http://minio:9000 -volatile.createBucket=true +default.key=18eb719ebffb4cd0ab78f9343f8aedd2 +default.secret=e7b6178dd61d4e0dbbc37ff7cb941aed +default.url=https://isti-cloud.isti.cnr.it:13808/ +default.createBucket=false +volatile.bucketName=shub-volatile-dev +volatile.key=18eb719ebffb4cd0ab78f9343f8aedd2 +volatile.secret=e7b6178dd61d4e0dbbc37ff7cb941aed +volatile.url=https://isti-cloud.isti.cnr.it:13808/ gcube-minio.key=SHUBTEST gcube-minio.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY gcube-minio.url=http://minio:9000 diff --git a/pom.xml b/pom.xml index d9f5678..530ee78 100644 --- a/pom.xml +++ b/pom.xml @@ -248,16 +248,16 @@ org.glassfish.jersey.media jersey-media-multipart - - postgresql - postgresql - 9.1-901.jdbc4 - runtime - org.reflections reflections + + + org.postgresql + postgresql + 42.7.0 + com.google.guava guava @@ -271,7 +271,7 @@ org.tukaani xz - 1.5 + 1.5 org.glassfish.jersey.test-framework.providers @@ -337,12 +337,18 @@ storage-manager-wrapper [4.0.0-SNAPSHOT,5.0.0-SNAPSHOT) - + + + com.amazonaws + aws-java-sdk-s3 + 1.12.512 + + org.gcube.common storagehub-client-library diff --git a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java index 65e6e4c..4e14bd2 100644 --- a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java +++ b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java @@ -1,7 +1,6 @@ package org.gcube.data.access.storagehub.storage.backend.impl; import java.io.InputStream; -import java.security.InvalidKeyException; import java.util.HashMap; import java.util.Map; import java.util.function.Function; @@ -14,16 +13,15 @@ import org.gcube.common.storagehub.model.storages.StorageBackend; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.minio.BucketExistsArgs; -import io.minio.CopyObjectArgs; -import io.minio.CopySource; -import io.minio.GetObjectArgs; -import io.minio.MakeBucketArgs; -import io.minio.MinioClient; -import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; -import io.minio.StatObjectArgs; -import io.minio.StatObjectResponse; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.S3ClientOptions; +import com.amazonaws.services.s3.model.ObjectMetadata; + public class S3Backend extends StorageBackend{ @@ -31,9 +29,9 @@ public class S3Backend extends StorageBackend{ Function keyGenerator; String bucketName; - MinioClient client; + AmazonS3 client; - private static final long PART_SIZE = 100000000; + //private static final long PART_SIZE = 100000000; @Override @@ -51,16 +49,19 @@ public class S3Backend extends StorageBackend{ String url = (String)parameters.get("url"); boolean createBucket = Boolean.valueOf((String)parameters.get("createBucket")); - log.debug("parameters are: backetName = {}, url = {}, createBucket = {}",this.bucketName, url, createBucket); + log.debug("parameters are: bucketName = {}, url = {}, createBucket = {}",this.bucketName, url, createBucket); try { - client = - MinioClient.builder() - .endpoint(url) - .credentials(accessKey, secret) - .build(); - if (createBucket && !client.bucketExists(BucketExistsArgs.builder().bucket(this.bucketName).build())) { - client.makeBucket(MakeBucketArgs.builder().bucket(this.bucketName).build()); + AWSCredentials credentials = new BasicAWSCredentials(accessKey, secret); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProtocol(Protocol.HTTPS); + + client = new AmazonS3Client(credentials, clientConfig); + client.setEndpoint(url); + client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); + + if (createBucket && !client.doesBucketExistV2(bucketName)) { + client.createBucket(bucketName); log.debug("bucket {} created",this.bucketName); } @@ -74,7 +75,7 @@ public class S3Backend extends StorageBackend{ boolean toReturn = true; try { - client.bucketExists(BucketExistsArgs.builder().bucket(this.bucketName).build()); + client.doesBucketExistV2(bucketName); }catch (Exception e) { log.error("error checking aliveness",e); toReturn = false; @@ -88,12 +89,8 @@ public class S3Backend extends StorageBackend{ String sourceKey = content.getStorageId(); String destinationKey = keyGenerator.apply(null); - try { - CopySource source = CopySource.builder().bucket(bucketName).object(sourceKey).build(); - // Copy the object into a new object in the same bucket. - CopyObjectArgs copyObjRequest = CopyObjectArgs.builder().source(source).bucket(bucketName).object(destinationKey).build(); - client.copyObject(copyObjRequest); + client.copyObject(bucketName, sourceKey, bucketName, destinationKey); } catch (Exception e) { throw new RuntimeException("error copying file on s3", e); } @@ -109,7 +106,7 @@ public class S3Backend extends StorageBackend{ @Override public void delete(String storageId) { try { - client.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(storageId).build()); + client.deleteObject(bucketName, storageId); } catch (Exception e) { throw new RuntimeException("error deleting file on s3", e); } @@ -131,26 +128,29 @@ public class S3Backend extends StorageBackend{ public MetaInfo upload(InputStream stream, String relativePath, String name, String storageId, Long size, String user) { try { - Map headers = new HashMap<>(); + /*Map headers = new HashMap<>(); headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY"); - Map userMetadata = new HashMap<>(); - userMetadata.put("user", user); - userMetadata.put("title", name); - - client.putObject( - PutObjectArgs.builder().bucket(bucketName).object(storageId).stream( - stream, size == null || size<=0?-1:size, size == null || size<=0?PART_SIZE:-1) - .headers(headers).userMetadata(userMetadata) - .build()); + Map userMetadata = new HashMap<>();*/ + ObjectMetadata objMeta = new ObjectMetadata(); + objMeta.addUserMetadata("user", user); + objMeta.addUserMetadata("title", name); + if (size != null && size >0) { + objMeta.setContentLength(size); + log.info("content length set to {}",size); + } else + log.info("content length not set"); + + log.info("uploading file {} with id {} in bucket {} ",name, storageId, bucketName); + + client.putObject(bucketName, storageId, stream, objMeta); long fileSize; if (size != null && size>0) fileSize = size; - else { - StatObjectResponse resp = client.statObject(StatObjectArgs.builder().bucket(bucketName).object(storageId).build()); - fileSize = resp.size(); - } + else + fileSize = client.getObjectMetadata(bucketName, storageId).getContentLength(); + return new MetaInfo(fileSize,storageId, null, getPayloadConfiguration()); } catch (Exception e) { @@ -163,11 +163,8 @@ public class S3Backend extends StorageBackend{ @Override public InputStream download(String id) throws StorageIdNotFoundException{ try { - InputStream inputStream = client.getObject(GetObjectArgs.builder().bucket(bucketName).object(id).build()); + InputStream inputStream = client.getObject(bucketName, id).getObjectContent(); return inputStream; - }catch (InvalidKeyException e) { - log.error("key %s not found", id); - throw new StorageIdNotFoundException(id, this.getPayloadConfiguration().getStorageName()); }catch (Exception e) { log.error("error downloading file form s3"); throw new RuntimeException("error downloading file from s3",e); @@ -183,12 +180,10 @@ public class S3Backend extends StorageBackend{ public Map getFileMetadata(String id) { try { - StatObjectResponse objectStat = - client.statObject( - StatObjectArgs.builder().bucket(bucketName).object(id).build()); - Map userMetadata = objectStat.userMetadata(); + ObjectMetadata objMeta = client.getObjectMetadata(bucketName, id); + Map userMetadata = objMeta.getUserMetadata(); HashMap toReturn = new HashMap<>(userMetadata); - toReturn.put("size" , Long.toString(objectStat.size())); + toReturn.put("size" , Long.toString(objMeta.getContentLength())); return toReturn; } catch (Exception e) { log.error("error getting metadata from s3"); diff --git a/src/main/resources/storage-settings.properties b/src/main/resources/storage-settings.properties index 7673907..012c755 100644 --- a/src/main/resources/storage-settings.properties +++ b/src/main/resources/storage-settings.properties @@ -1,12 +1,12 @@ default.bucketName=storagehub-dev -default.key=qa4zD0QqmcG0JZNG -default.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL -default.url=https://minio.d4science.org/ +default.key=68c40352df6e418cbe7ea331d5a95c5b +default.secret=N-wuiTlRv9uJjoa86faJIMgK_dzpc1wccD_pRiwG6--M0xkrat_t-Lmg0DW5MYefOvHfkE8bep_6V9HpOF7HRQ +default.url=https://isti-cloud.isti.cnr.it:13808/swift/v1/AUTH_e8f8ca72f30648a8b389b4e745ac83a9/ default.createBucket=false volatile.bucketName=shub-volatile-dev -volatile.key=qa4zD0QqmcG0JZNG -volatile.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL -volatile.url=https://minio.d4science.org/ +volatile.key=68c40352df6e418cbe7ea331d5a95c5b +volatile.secret=N-wuiTlRv9uJjoa86faJIMgK_dzpc1wccD_pRiwG6--M0xkrat_t-Lmg0DW5MYefOvHfkE8bep_6V9HpOF7HRQ +volatile.url=https://isti-cloud.isti.cnr.it:13808/swift/v1/AUTH_e8f8ca72f30648a8b389b4e745ac83a9/ volatile.createBucket=false gcube-minio.key=qa4zD0QqmcG0JZNG gcube-minio.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL