From 4878f8bb183a7a18fac57fabe64e33bdf20e52e9 Mon Sep 17 00:00:00 2001 From: lucio Date: Mon, 25 Mar 2024 16:37:40 +0100 Subject: [PATCH] updated --- .settings/org.eclipse.wst.common.component | 60 ++++++++++++------- Dockerfile | 8 +-- buildImageAndStart.sh | 4 +- docker-compose.yml | 2 +- docker/storage-settings.properties | 27 +++------ pom.xml | 51 +++++++--------- .../storagehub/handlers/GroupHandler.java | 19 ++++++ .../handlers/items/ItemHandler.java | 1 - .../storagehub/services/GroupManager.java | 2 - .../GCubeVolatileStorageBackendFactory.java | 37 +++++++----- .../GcubeDefaultS3StorageBackendFactory.java | 8 ++- .../storage/backend/impl/S3Backend.java | 15 +++-- .../data/access/fs/external/CreateUsers.java | 5 +- 13 files changed, 133 insertions(+), 106 deletions(-) diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index f2e222e..380cb7e 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -1,5 +1,6 @@ - + + @@ -39,7 +40,8 @@ - + + @@ -79,7 +81,8 @@ - + + @@ -119,49 +122,63 @@ - + + - + + - + + uses - + + - + + - + + uses - + + - + + - + + uses - + + - + + - + + uses - + + - + + @@ -201,7 +218,8 @@ - + + @@ -241,7 +259,8 @@ - + + @@ -281,7 +300,8 @@ - + + diff --git a/Dockerfile b/Dockerfile index b046175..9c71345 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,10 @@ FROM d4science/smartgears-distribution:4.0.0-SNAPSHOT-java17-tomcat10.1.19 ARG REPOUSER=admin ARG REPOPWD=admin - #install unzip - RUN apt-get update && apt-get install unzip - COPY ./target/storagehub.war /tomcat/webapps/ COPY ./docker/jackrabbit /app/jackrabbit COPY ./docker/storagehub.xml /tomcat/conf/Catalina/localhost/ COPY ./docker/logback.xml /etc/ COPY ./docker/container.ini /etc/ - RUN unzip /tomcat/webapps/storagehub.war -d /tomcat/webapps/storagehub - RUN rm /tomcat/webapps/storagehub.war - COPY ./docker/storage-settings.properties /tomcat/webapps/storagehub/WEB-INF/classes/ \ No newline at end of file + RUN mkdir -p /etc/config/StorageHub + COPY ./docker/storage-settings.properties /etc/config/StorageHub/ \ No newline at end of file diff --git a/buildImageAndStart.sh b/buildImageAndStart.sh index ce18d3c..18933df 100755 --- a/buildImageAndStart.sh +++ b/buildImageAndStart.sh @@ -1,7 +1,9 @@ #!/bin/bash +set -e + NAME=storagehub -PORT=8080 +PORT=8100 DEBUG_PORT=5005 debug=false compile=false diff --git a/docker-compose.yml b/docker-compose.yml index 4c4ebc8..ae81e77 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: environment: _JAVA_OPTIONS: -Xdebug - -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005 + -agentlib:jdwp=transport=dt_socket,server=y,suspend=${SUSPEND:-n},address=*:5005 ports: - '8081:8080' - '5005:5005' diff --git a/docker/storage-settings.properties b/docker/storage-settings.properties index a3268e6..5ca0c23 100644 --- a/docker/storage-settings.properties +++ b/docker/storage-settings.properties @@ -1,19 +1,10 @@ -#default.bucketName=storagehub-dev -#default.key=SHUBTEST -#default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY -#default.url=http://minio:9000 -#default.createBucket=true -#volatile.bucketName=storagehub-volatile-dev -#volatile.key=SHUBTEST -#volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY -#volatile.url=http://minio:9000 -#volatile.createBucket=true default.bucketName=storagehub-dev -default.key=18eb719ebffb4cd0ab78f9343f8aedd2 -default.secret=e7b6178dd61d4e0dbbc37ff7cb941aed -default.url=https://isti-cloud.isti.cnr.it:13808/ -default.createBucket=false -volatile.bucketName=shub-volatile-dev -volatile.key=18eb719ebffb4cd0ab78f9343f8aedd2 -volatile.secret=e7b6178dd61d4e0dbbc37ff7cb941aed -volatile.url=https://isti-cloud.isti.cnr.it:13808/ \ No newline at end of file +default.key=SHUBTEST +default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY +default.url=http://minio:9000/ +default.createBucket=true +volatile.bucketName=storagehub-volatile-dev +volatile.key=SHUBTEST +volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY +volatile.url=http://minio:9000/ +volatile.createBucket=true \ No newline at end of file diff --git a/pom.xml b/pom.xml index bd5b546..a2d1e0e 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 2.15.3 2.0.12 2.6.0 - 1.14.0 + 1.15.0 ${project.basedir}/distro REST web service for Jackrabbit storagehub @@ -113,6 +113,7 @@ org.gcube.common gxJRS + javax.jcr @@ -334,7 +335,12 @@ com.amazonaws aws-java-sdk-s3 - 1.12.512 + 1.12.683 + + + javax.xml.bind + jaxb-api + 2.3.1 diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java b/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java index bd2c187..859a8d6 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java @@ -7,10 +7,14 @@ import javax.jcr.Node; import javax.jcr.NodeIterator; import javax.jcr.PathNotFoundException; import javax.jcr.RepositoryException; +import javax.jcr.security.AccessControlEntry; +import javax.jcr.security.AccessControlManager; import org.apache.jackrabbit.api.JackrabbitSession; +import org.apache.jackrabbit.api.security.JackrabbitAccessControlList; import org.apache.jackrabbit.api.security.user.Group; import org.apache.jackrabbit.api.security.user.User; +import org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils; import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters; import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.common.storagehub.model.types.NodeProperty; @@ -42,6 +46,21 @@ public class GroupHandler { Node folder = getFolderNodeRelatedToGroup(session, folderName); + //Removing the ACL for the user + AccessControlManager acm = session.getAccessControlManager(); + JackrabbitAccessControlList acls = AccessControlUtils.getAccessControlList(acm, folder.getPath()); + AccessControlEntry entryToDelete= null; + for (AccessControlEntry ace :acls.getAccessControlEntries()) { + if (ace.getPrincipal().getName().equals(userId)) { + entryToDelete = ace; + break; + } + + } + if (entryToDelete!=null) + acls.removeAccessControlEntry(entryToDelete); + + boolean found = false; NodeIterator ni = folder.getSharedSet(); while (ni.hasNext()) { diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java b/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java index 5db32ed..beef969 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java @@ -143,7 +143,6 @@ public class ItemHandler { params.getDescription(), params.getFileDetails(), params.getUser(), true); params.getSession().save(); versionHandler.checkinContentNode(newNode); - log.info("file with id {} correctly created", newNode.getIdentifier()); return newNode; } diff --git a/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java b/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java index dca6e02..c924e63 100644 --- a/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java @@ -464,8 +464,6 @@ public class GroupManager { session = (JackrabbitSession) repository.getRepository().login(Constants.JCR_CREDENTIALS); - //TODO: remove also acls of the user removed - success = groupHandler.removeUserFromGroup(groupId, userId, session); session.save(); diff --git a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeVolatileStorageBackendFactory.java b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeVolatileStorageBackendFactory.java index 2f4eb1a..24df1a3 100644 --- a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeVolatileStorageBackendFactory.java +++ b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeVolatileStorageBackendFactory.java @@ -1,7 +1,9 @@ package org.gcube.data.access.storagehub.storage.backend.impl; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -15,45 +17,48 @@ import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters; import org.gcube.common.storagehub.model.items.nodes.PayloadBackend; import org.gcube.common.storagehub.model.storages.StorageBackend; import org.gcube.common.storagehub.model.storages.StorageBackendFactory; - - +import org.gcube.smartgears.ContextProvider; +import org.gcube.smartgears.context.application.ApplicationContext; @Singleton public class GCubeVolatileStorageBackendFactory implements StorageBackendFactory { private StorageBackend singleton; - private static final String PROP_PREFIX = "volatile."; + private static final String PROP_PREFIX = "volatile."; public static final String NAME = "volatile-minio"; - + @PostConstruct - public void init(){ - S3Backend s3Backend = new S3Backend(new PayloadBackend(getName(), getParameters()), (String) -> UUID.randomUUID().toString()); - s3Backend.setPayloadConfiguration(new PayloadBackend(getName(),null)); + public void init() { + S3Backend s3Backend = new S3Backend(new PayloadBackend(getName(), getParameters()), + (String) -> UUID.randomUUID().toString()); + s3Backend.setPayloadConfiguration(new PayloadBackend(getName(), null)); this.singleton = s3Backend; } @Override public String getName() { - return NAME; + return NAME; } @Override public boolean isSystemStorage() { return true; } - + @Override public StorageBackend create(PayloadBackend payloadConfiguration) throws InvalidCallParameters { - if (payloadConfiguration.getParameters().isEmpty()) + if (payloadConfiguration.getParameters().isEmpty()) return singleton; throw new InvalidCallParameters("gcubeMinIo created with not empty parameters"); } - private Metadata getParameters(){ - try (InputStream input = GCubeVolatileStorageBackendFactory.class.getClassLoader().getResourceAsStream("storage-settings.properties")) { + private Metadata getParameters() { + ApplicationContext context = ContextProvider.get(); + String folderPath = context.appSpecificConfigurationFolder().toString(); + try (InputStream input = new FileInputStream(Paths.get(folderPath, "storage-settings.properties").toFile())) { Properties prop = new Properties(); @@ -61,15 +66,17 @@ public class GCubeVolatileStorageBackendFactory implements StorageBackendFactory Map params = new HashMap(); - prop.forEach((k,v) -> { if (k.toString().startsWith(PROP_PREFIX)) params.put(k.toString().replace(PROP_PREFIX, ""), v.toString());}); - + prop.forEach((k, v) -> { + if (k.toString().startsWith(PROP_PREFIX)) + params.put(k.toString().replace(PROP_PREFIX, ""), v.toString()); + }); + return new Metadata(params); } catch (IOException ex) { throw new RuntimeException("error initializing MinIO", ex); } - } } \ No newline at end of file diff --git a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeDefaultS3StorageBackendFactory.java b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeDefaultS3StorageBackendFactory.java index 241d4b5..f7e4a59 100644 --- a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeDefaultS3StorageBackendFactory.java +++ b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeDefaultS3StorageBackendFactory.java @@ -1,7 +1,9 @@ package org.gcube.data.access.storagehub.storage.backend.impl; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -16,6 +18,8 @@ import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters; import org.gcube.common.storagehub.model.items.nodes.PayloadBackend; import org.gcube.common.storagehub.model.storages.StorageBackend; import org.gcube.common.storagehub.model.storages.StorageBackendFactory; +import org.gcube.smartgears.ContextProvider; +import org.gcube.smartgears.context.application.ApplicationContext; @Singleton public class GcubeDefaultS3StorageBackendFactory implements StorageBackendFactory { @@ -50,7 +54,9 @@ public class GcubeDefaultS3StorageBackendFactory implements StorageBackendFactor } private Metadata getParameters(){ - try (InputStream input = GcubeDefaultS3StorageBackendFactory.class.getClassLoader().getResourceAsStream("storage-settings.properties")) { + ApplicationContext context = ContextProvider.get(); + String folderPath = context.appSpecificConfigurationFolder().toString(); + try (InputStream input = new FileInputStream(Paths.get(folderPath, "storage-settings.properties").toFile())) { Properties prop = new Properties(); diff --git a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java index 477dbbc..210e820 100644 --- a/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java +++ b/src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java @@ -18,9 +18,9 @@ import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.S3ClientOptions; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -52,16 +52,18 @@ public class S3Backend extends StorageBackend{ try { AWSCredentials credentials = new BasicAWSCredentials(accessKey, secret); ClientConfiguration clientConfig = new ClientConfiguration(); - clientConfig.setProtocol(Protocol.HTTPS); + clientConfig.setProtocol(Protocol.HTTPS); client = AmazonS3ClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(credentials)).withClientConfiguration(clientConfig).build(); + .withEndpointConfiguration(new EndpointConfiguration(url,"us-east-1" )) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .enablePathStyleAccess() + .withClientConfiguration(clientConfig).build(); - client.setEndpoint(url); - client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); if (createBucket && !client.doesBucketExistV2(bucketName)) { + log.debug("creating {} bucket",this.bucketName); client.createBucket(bucketName); log.debug("bucket {} created",this.bucketName); } @@ -129,9 +131,6 @@ public class S3Backend extends StorageBackend{ public MetaInfo upload(InputStream stream, String relativePath, String name, String storageId, Long size, String user) { try { - /*Map headers = new HashMap<>(); - headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY"); - Map userMetadata = new HashMap<>();*/ ObjectMetadata objMeta = new ObjectMetadata(); objMeta.addUserMetadata("user", user); objMeta.addUserMetadata("title", name); diff --git a/src/test/java/org/gcube/data/access/fs/external/CreateUsers.java b/src/test/java/org/gcube/data/access/fs/external/CreateUsers.java index a5a5c6c..182c1fc 100644 --- a/src/test/java/org/gcube/data/access/fs/external/CreateUsers.java +++ b/src/test/java/org/gcube/data/access/fs/external/CreateUsers.java @@ -155,13 +155,14 @@ public class CreateUsers { public void createUser2AndAddItToVres() throws Exception{ client.impersonate(user); client.createUserAccount(user2); - /*client.getVreFolderManager(vreWA).addUser(user2); + client.getVreFolderManager(vreWA).addUser(user2); client.getVreFolderManager(vreRO).addUser(user2); - client.getVreFolderManager(vreWO).addUser(user2);*/ + client.getVreFolderManager(vreWO).addUser(user2); } @Test public void printVREFoldersAdmin() throws StorageHubException{ + System.out.println("launching ....." ); List vreFolders = client.getVREFolders().getContainers().stream().map(c -> ((FolderContainer) c)).collect(Collectors.toList()); for (FolderContainer fc : vreFolders) { System.out.println(fc.get().getTitle());