gCubeSystem
/
storagehub
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

jdbc driver updated

This commit is contained in:
lucio 2023-12-01 13:04:07 +01:00
parent 2452a25349
commit f75f7d86d9
6 changed files with 114 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.settings/org.eclipse.jdt.core.prefs
View File

@ -7,5 +7,5 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=enabled org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=11 org.eclipse.jdt.core.compiler.source=11

44
.settings/org.eclipse.wst.common.component
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?><project-modules id="moduleCoreId" project-version="1.5.0"> <?xml version="1.0" encoding="UTF-8"?><project-modules id="moduleCoreId" project-version="1.5.0">
@ -29,7 +31,9 @@
<wb-module deploy-name="storagehub"> <wb-module deploy-name="storagehub">
@ -59,7 +63,9 @@
<wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/> <wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/>
@ -89,7 +95,9 @@
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/> <wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
@ -119,23 +127,21 @@
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/> <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/> <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<dependent-module archiveName="common-smartgears-app-3.0.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/common-smartgears-app/common-smartgears-app"> <dependent-module archiveName="common-scope-2.0.0.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/common-scope/common-scope">
<dependency-type>uses</dependency-type> <dependency-type>uses</dependency-type>
</dependent-module> </dependent-module>
<dependent-module archiveName="common-scope-2.0.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/common-scope/common-scope">
<dependency-type>uses</dependency-type>
</dependent-module>
<dependent-module archiveName="storagehub-model-2.0.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/storagehub-model/storagehub-model">
<dependency-type>uses</dependency-type>
</dependent-module>
@ -165,7 +171,9 @@
<property name="context-root" value="storagehub"/> <property name="context-root" value="storagehub"/>
@ -195,7 +203,9 @@
<property name="java-output-path" value="/storagehub-webapp_BRANCH/target/classes"/> <property name="java-output-path" value="/storagehub-webapp_BRANCH/target/classes"/>
@ -225,7 +235,9 @@
</wb-module> </wb-module>

27
docker/storage-settings.properties
View File

@ -1,13 +1,22 @@
#default.bucketName=storagehub-dev
#default.key=SHUBTEST
#default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
#default.url=http://minio:9000
#default.createBucket=true
#volatile.bucketName=storagehub-volatile-dev
#volatile.key=SHUBTEST
#volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
#volatile.url=http://minio:9000
#volatile.createBucket=true
default.bucketName=storagehub-dev default.bucketName=storagehub-dev
default.key=SHUBTEST default.key=18eb719ebffb4cd0ab78f9343f8aedd2
default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY default.secret=e7b6178dd61d4e0dbbc37ff7cb941aed
default.url=http://minio:9000 default.url=https://isti-cloud.isti.cnr.it:13808/
default.createBucket=true default.createBucket=false
volatile.bucketName=storagehub-volatile-dev volatile.bucketName=shub-volatile-dev
volatile.key=SHUBTEST volatile.key=18eb719ebffb4cd0ab78f9343f8aedd2
volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY volatile.secret=e7b6178dd61d4e0dbbc37ff7cb941aed
volatile.url=http://minio:9000 volatile.url=https://isti-cloud.isti.cnr.it:13808/
volatile.createBucket=true
gcube-minio.key=SHUBTEST gcube-minio.key=SHUBTEST
gcube-minio.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY gcube-minio.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
gcube-minio.url=http://minio:9000 gcube-minio.url=http://minio:9000

24
pom.xml
View File

@ -248,16 +248,16 @@
<groupId>org.glassfish.jersey.media</groupId> <groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-multipart</artifactId> <artifactId>jersey-media-multipart</artifactId>
</dependency> </dependency>
<dependency>
<groupId>postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>9.1-901.jdbc4</version>
<scope>runtime</scope>
</dependency>
<dependency> <dependency>
<groupId>org.reflections</groupId> <groupId>org.reflections</groupId>
<artifactId>reflections</artifactId> <artifactId>reflections</artifactId>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.7.0</version>
</dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>
@ -271,7 +271,7 @@
<dependency> <dependency>
<groupId>org.tukaani</groupId> <groupId>org.tukaani</groupId>
<artifactId>xz</artifactId> <artifactId>xz</artifactId>
<version>1.5</version> <version>1.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.glassfish.jersey.test-framework.providers</groupId> <groupId>org.glassfish.jersey.test-framework.providers</groupId>
@ -337,12 +337,18 @@
<artifactId>storage-manager-wrapper</artifactId> <artifactId>storage-manager-wrapper</artifactId>
<version>[4.0.0-SNAPSHOT,5.0.0-SNAPSHOT)</version> <version>[4.0.0-SNAPSHOT,5.0.0-SNAPSHOT)</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/io.minio/minio --> <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3 -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.512</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.minio/minio
<dependency> <dependency>
<groupId>io.minio</groupId> <groupId>io.minio</groupId>
<artifactId>minio</artifactId> <artifactId>minio</artifactId>
<version>8.3.3</version> <version>8.3.3</version>
</dependency> </dependency> -->
<dependency> <dependency>
<groupId>org.gcube.common</groupId> <groupId>org.gcube.common</groupId>
<artifactId>storagehub-client-library</artifactId> <artifactId>storagehub-client-library</artifactId>

97
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java
View File

@ -1,7 +1,6 @@
package org.gcube.data.access.storagehub.storage.backend.impl; package org.gcube.data.access.storagehub.storage.backend.impl;
import java.io.InputStream; import java.io.InputStream;
import java.security.InvalidKeyException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.function.Function; import java.util.function.Function;
@ -14,16 +13,15 @@ import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import io.minio.BucketExistsArgs; import com.amazonaws.ClientConfiguration;
import io.minio.CopyObjectArgs; import com.amazonaws.Protocol;
import io.minio.CopySource; import com.amazonaws.auth.AWSCredentials;
import io.minio.GetObjectArgs; import com.amazonaws.auth.BasicAWSCredentials;
import io.minio.MakeBucketArgs; import com.amazonaws.services.s3.AmazonS3;
import io.minio.MinioClient; import com.amazonaws.services.s3.AmazonS3Client;
import io.minio.PutObjectArgs; import com.amazonaws.services.s3.S3ClientOptions;
import io.minio.RemoveObjectArgs; import com.amazonaws.services.s3.model.ObjectMetadata;
import io.minio.StatObjectArgs;
import io.minio.StatObjectResponse;
public class S3Backend extends StorageBackend{ public class S3Backend extends StorageBackend{
@ -31,9 +29,9 @@ public class S3Backend extends StorageBackend{
Function<Void, String> keyGenerator; Function<Void, String> keyGenerator;
String bucketName; String bucketName;
MinioClient client; AmazonS3 client;
private static final long PART_SIZE = 100000000; //private static final long PART_SIZE = 100000000;
@Override @Override
@ -51,16 +49,19 @@ public class S3Backend extends StorageBackend{
String url = (String)parameters.get("url"); String url = (String)parameters.get("url");
boolean createBucket = Boolean.valueOf((String)parameters.get("createBucket")); boolean createBucket = Boolean.valueOf((String)parameters.get("createBucket"));
log.debug("parameters are: backetName = {}, url = {}, createBucket = {}",this.bucketName, url, createBucket); log.debug("parameters are: bucketName = {}, url = {}, createBucket = {}",this.bucketName, url, createBucket);
try { try {
client = AWSCredentials credentials = new BasicAWSCredentials(accessKey, secret);
MinioClient.builder() ClientConfiguration clientConfig = new ClientConfiguration();
.endpoint(url) clientConfig.setProtocol(Protocol.HTTPS);
.credentials(accessKey, secret)
.build(); client = new AmazonS3Client(credentials, clientConfig);
if (createBucket && !client.bucketExists(BucketExistsArgs.builder().bucket(this.bucketName).build())) { client.setEndpoint(url);
client.makeBucket(MakeBucketArgs.builder().bucket(this.bucketName).build()); client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
if (createBucket && !client.doesBucketExistV2(bucketName)) {
client.createBucket(bucketName);
log.debug("bucket {} created",this.bucketName); log.debug("bucket {} created",this.bucketName);
} }
@ -74,7 +75,7 @@ public class S3Backend extends StorageBackend{
boolean toReturn = true; boolean toReturn = true;
try { try {
client.bucketExists(BucketExistsArgs.builder().bucket(this.bucketName).build()); client.doesBucketExistV2(bucketName);
}catch (Exception e) { }catch (Exception e) {
log.error("error checking aliveness",e); log.error("error checking aliveness",e);
toReturn = false; toReturn = false;
@ -88,12 +89,8 @@ public class S3Backend extends StorageBackend{
String sourceKey = content.getStorageId(); String sourceKey = content.getStorageId();
String destinationKey = keyGenerator.apply(null); String destinationKey = keyGenerator.apply(null);
try { try {
CopySource source = CopySource.builder().bucket(bucketName).object(sourceKey).build(); client.copyObject(bucketName, sourceKey, bucketName, destinationKey);
// Copy the object into a new object in the same bucket.
CopyObjectArgs copyObjRequest = CopyObjectArgs.builder().source(source).bucket(bucketName).object(destinationKey).build();
client.copyObject(copyObjRequest);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("error copying file on s3", e); throw new RuntimeException("error copying file on s3", e);
} }
@ -109,7 +106,7 @@ public class S3Backend extends StorageBackend{
@Override @Override
public void delete(String storageId) { public void delete(String storageId) {
try { try {
client.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(storageId).build()); client.deleteObject(bucketName, storageId);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("error deleting file on s3", e); throw new RuntimeException("error deleting file on s3", e);
} }
@ -131,26 +128,29 @@ public class S3Backend extends StorageBackend{
public MetaInfo upload(InputStream stream, String relativePath, String name, String storageId, Long size, String user) { public MetaInfo upload(InputStream stream, String relativePath, String name, String storageId, Long size, String user) {
try { try {
Map<String, String> headers = new HashMap<>(); /*Map<String, String> headers = new HashMap<>();
headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY"); headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY");
Map<String, String> userMetadata = new HashMap<>(); Map<String, String> userMetadata = new HashMap<>();*/
userMetadata.put("user", user); ObjectMetadata objMeta = new ObjectMetadata();
userMetadata.put("title", name); objMeta.addUserMetadata("user", user);
objMeta.addUserMetadata("title", name);
client.putObject(
PutObjectArgs.builder().bucket(bucketName).object(storageId).stream(
stream, size == null || size<=0?-1:size, size == null || size<=0?PART_SIZE:-1)
.headers(headers).userMetadata(userMetadata)
.build());
if (size != null && size >0) {
objMeta.setContentLength(size);
log.info("content length set to {}",size);
} else
log.info("content length not set");
log.info("uploading file {} with id {} in bucket {} ",name, storageId, bucketName);
client.putObject(bucketName, storageId, stream, objMeta);
long fileSize; long fileSize;
if (size != null && size>0) if (size != null && size>0)
fileSize = size; fileSize = size;
else { else
StatObjectResponse resp = client.statObject(StatObjectArgs.builder().bucket(bucketName).object(storageId).build()); fileSize = client.getObjectMetadata(bucketName, storageId).getContentLength();
fileSize = resp.size();
}
return new MetaInfo(fileSize,storageId, null, getPayloadConfiguration()); return new MetaInfo(fileSize,storageId, null, getPayloadConfiguration());
} catch (Exception e) { } catch (Exception e) {
@ -163,11 +163,8 @@ public class S3Backend extends StorageBackend{
@Override @Override
public InputStream download(String id) throws StorageIdNotFoundException{ public InputStream download(String id) throws StorageIdNotFoundException{
try { try {
InputStream inputStream = client.getObject(GetObjectArgs.builder().bucket(bucketName).object(id).build()); InputStream inputStream = client.getObject(bucketName, id).getObjectContent();
return inputStream; return inputStream;
}catch (InvalidKeyException e) {
log.error("key %s not found", id);
throw new StorageIdNotFoundException(id, this.getPayloadConfiguration().getStorageName());
}catch (Exception e) { }catch (Exception e) {
log.error("error downloading file form s3"); log.error("error downloading file form s3");
throw new RuntimeException("error downloading file from s3",e); throw new RuntimeException("error downloading file from s3",e);
@ -183,12 +180,10 @@ public class S3Backend extends StorageBackend{
public Map<String, String> getFileMetadata(String id) { public Map<String, String> getFileMetadata(String id) {
try { try {
StatObjectResponse objectStat = ObjectMetadata objMeta = client.getObjectMetadata(bucketName, id);
client.statObject( Map<String, String> userMetadata = objMeta.getUserMetadata();
StatObjectArgs.builder().bucket(bucketName).object(id).build());
Map<String, String> userMetadata = objectStat.userMetadata();
HashMap<String, String> toReturn = new HashMap<>(userMetadata); HashMap<String, String> toReturn = new HashMap<>(userMetadata);
toReturn.put("size" , Long.toString(objectStat.size())); toReturn.put("size" , Long.toString(objMeta.getContentLength()));
return toReturn; return toReturn;
} catch (Exception e) { } catch (Exception e) {
log.error("error getting metadata from s3"); log.error("error getting metadata from s3");

12
src/main/resources/storage-settings.properties
View File

@ -1,12 +1,12 @@
default.bucketName=storagehub-dev default.bucketName=storagehub-dev
default.key=qa4zD0QqmcG0JZNG default.key=68c40352df6e418cbe7ea331d5a95c5b
default.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL default.secret=N-wuiTlRv9uJjoa86faJIMgK_dzpc1wccD_pRiwG6--M0xkrat_t-Lmg0DW5MYefOvHfkE8bep_6V9HpOF7HRQ
default.url=https://minio.d4science.org/ default.url=https://isti-cloud.isti.cnr.it:13808/swift/v1/AUTH_e8f8ca72f30648a8b389b4e745ac83a9/
default.createBucket=false default.createBucket=false
volatile.bucketName=shub-volatile-dev volatile.bucketName=shub-volatile-dev
volatile.key=qa4zD0QqmcG0JZNG volatile.key=68c40352df6e418cbe7ea331d5a95c5b
volatile.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL volatile.secret=N-wuiTlRv9uJjoa86faJIMgK_dzpc1wccD_pRiwG6--M0xkrat_t-Lmg0DW5MYefOvHfkE8bep_6V9HpOF7HRQ
volatile.url=https://minio.d4science.org/ volatile.url=https://isti-cloud.isti.cnr.it:13808/swift/v1/AUTH_e8f8ca72f30648a8b389b4e745ac83a9/
volatile.createBucket=false volatile.createBucket=false
gcube-minio.key=qa4zD0QqmcG0JZNG gcube-minio.key=qa4zD0QqmcG0JZNG
gcube-minio.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL gcube-minio.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL