gCubeSystem
/
storagehub
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

updated

This commit is contained in:
lucio 2024-03-25 16:37:40 +01:00
parent eefa46cbf6
commit 4878f8bb18
13 changed files with 133 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
.settings/org.eclipse.wst.common.component
View File

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?><project-modules id="moduleCoreId" project-version="1.5.0">
@ -39,7 +40,8 @@
<wb-module deploy-name="storagehub">
@ -79,7 +81,8 @@
<wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/>
@ -119,49 +122,63 @@
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/test/java"/>
<dependent-module archiveName="common-smartgears-app-3.0.1-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/common-smartgears-app/common-smartgears-app">
<dependency-type>uses</dependency-type>
</dependent-module>
<dependent-module archiveName="authorization-control-library-2.0.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/authorization-control-library/authorization-control-library">
<dependency-type>uses</dependency-type>
</dependent-module>
<dependent-module archiveName="storagehub-model-2.0.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/storagehub-model/storagehub-model">
<dependency-type>uses</dependency-type>
</dependent-module>
<dependent-module archiveName="storagehub-script-utils-2.0.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/storagehub-scripting-util/storagehub-scripting-util">
<dependency-type>uses</dependency-type>
</dependent-module>
@ -201,7 +218,8 @@
<property name="context-root" value="storagehub"/>
@ -241,7 +259,8 @@
<property name="java-output-path" value="/storagehub-webapp_BRANCH/target/classes"/>
@ -281,7 +300,8 @@
</wb-module>

8
Dockerfile
View File

@ -2,14 +2,10 @@ FROM d4science/smartgears-distribution:4.0.0-SNAPSHOT-java17-tomcat10.1.19
ARG REPOUSER=admin
ARG REPOPWD=admin
#install unzip
RUN apt-get update && apt-get install unzip
COPY ./target/storagehub.war /tomcat/webapps/
COPY ./docker/jackrabbit /app/jackrabbit
COPY ./docker/storagehub.xml /tomcat/conf/Catalina/localhost/
COPY ./docker/logback.xml /etc/
COPY ./docker/container.ini /etc/
RUN unzip /tomcat/webapps/storagehub.war -d /tomcat/webapps/storagehub
RUN rm /tomcat/webapps/storagehub.war
COPY ./docker/storage-settings.properties /tomcat/webapps/storagehub/WEB-INF/classes/
RUN mkdir -p /etc/config/StorageHub
COPY ./docker/storage-settings.properties /etc/config/StorageHub/

4
buildImageAndStart.sh
View File

@ -1,7 +1,9 @@
#!/bin/bash
set -e
NAME=storagehub
PORT=8080
PORT=8100
DEBUG_PORT=5005
debug=false
compile=false

2
docker-compose.yml
View File

@ -22,7 +22,7 @@ services:
environment:
_JAVA_OPTIONS:
-Xdebug
-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005
-agentlib:jdwp=transport=dt_socket,server=y,suspend=${SUSPEND:-n},address=*:5005
ports:
- '8081:8080'
- '5005:5005'

27
docker/storage-settings.properties
View File

@ -1,19 +1,10 @@
#default.bucketName=storagehub-dev
#default.key=SHUBTEST
#default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
#default.url=http://minio:9000
#default.createBucket=true
#volatile.bucketName=storagehub-volatile-dev
#volatile.key=SHUBTEST
#volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
#volatile.url=http://minio:9000
#volatile.createBucket=true
default.bucketName=storagehub-dev
default.key=18eb719ebffb4cd0ab78f9343f8aedd2
default.secret=e7b6178dd61d4e0dbbc37ff7cb941aed
default.url=https://isti-cloud.isti.cnr.it:13808/
default.createBucket=false
volatile.bucketName=shub-volatile-dev
volatile.key=18eb719ebffb4cd0ab78f9343f8aedd2
volatile.secret=e7b6178dd61d4e0dbbc37ff7cb941aed
volatile.url=https://isti-cloud.isti.cnr.it:13808/
default.key=SHUBTEST
default.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
default.url=http://minio:9000/
default.createBucket=true
volatile.bucketName=storagehub-volatile-dev
volatile.key=SHUBTEST
volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
volatile.url=http://minio:9000/
volatile.createBucket=true

51
pom.xml
View File

@ -26,7 +26,7 @@
<jackson.version>2.15.3</jackson.version>
<slf4j.version>2.0.12</slf4j.version>
<tika.version>2.6.0</tika.version>
<aspectj-plugin.version>1.14.0</aspectj-plugin.version>
<aspectj-plugin.version>1.15.0</aspectj-plugin.version>
<distroDirectory>${project.basedir}/distro</distroDirectory>
<description>REST web service for Jackrabbit</description>
<warname>storagehub</warname>
@ -113,6 +113,7 @@
<groupId>org.gcube.common</groupId>
<artifactId>gxJRS</artifactId>
</dependency>
<!-- JCR dependencies -->
<dependency>
<groupId>javax.jcr</groupId>
@ -334,7 +335,12 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.512</version>
<version>1.12.683</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.minio/minio
<dependency>
@ -375,39 +381,14 @@
</resources>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
<version>1.0.0</version>
<configuration>
<lifecycleMappingMetadata>
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
<groupId>org.codehaus.mojo</groupId>
<artifactId>aspectj-maven-plugin</artifactId>
<versionRange>[1.0,)</versionRange>
<goals>
<goal>test-compile</goal>
<goal>compile</goal>
</goals>
</pluginExecutionFilter>
<action>
<execute />
</action>
</pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>aspectj-maven-plugin</artifactId>
<version>${aspectj-plugin.version}</version>
<configuration>
<complianceLevel>11</complianceLevel>
<source>11</source>
<target>11</target>
<complianceLevel>17</complianceLevel>
<source>17</source>
<target>17</target>
<aspectLibraries>
<aspectLibrary>
<groupId>org.gcube.common</groupId>
@ -415,10 +396,18 @@
</aspectLibrary>
</aspectLibraries>
</configuration>
<dependencies>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjtools</artifactId>
<version>1.9.21.1</version>
</dependency>
</dependencies>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>compile</goal> <!-- use this goal to weave
all your main classes -->
</goals>
</execution>
</executions>

19
src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java
View File

@ -7,10 +7,14 @@ import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.PathNotFoundException;
import javax.jcr.RepositoryException;
import javax.jcr.security.AccessControlEntry;
import javax.jcr.security.AccessControlManager;
import org.apache.jackrabbit.api.JackrabbitSession;
import org.apache.jackrabbit.api.security.JackrabbitAccessControlList;
import org.apache.jackrabbit.api.security.user.Group;
import org.apache.jackrabbit.api.security.user.User;
import org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils;
import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.types.NodeProperty;
@ -42,6 +46,21 @@ public class GroupHandler {
Node folder = getFolderNodeRelatedToGroup(session, folderName);
//Removing the ACL for the user
AccessControlManager acm = session.getAccessControlManager();
JackrabbitAccessControlList acls = AccessControlUtils.getAccessControlList(acm, folder.getPath());
AccessControlEntry entryToDelete= null;
for (AccessControlEntry ace :acls.getAccessControlEntries()) {
if (ace.getPrincipal().getName().equals(userId)) {
entryToDelete = ace;
break;
}
}
if (entryToDelete!=null)
acls.removeAccessControlEntry(entryToDelete);
boolean found = false;
NodeIterator ni = folder.getSharedSet();
while (ni.hasNext()) {

1
src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java
View File

@ -143,7 +143,6 @@ public class ItemHandler {
params.getDescription(), params.getFileDetails(), params.getUser(), true);
params.getSession().save();
versionHandler.checkinContentNode(newNode);
log.info("file with id {} correctly created", newNode.getIdentifier());
return newNode;
}

2
src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java
View File

@ -464,8 +464,6 @@ public class GroupManager {
session = (JackrabbitSession) repository.getRepository().login(Constants.JCR_CREDENTIALS);
//TODO: remove also acls of the user removed
success = groupHandler.removeUserFromGroup(groupId, userId, session);
session.save();

37
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeVolatileStorageBackendFactory.java
View File

@ -1,7 +1,9 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@ -15,45 +17,48 @@ import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.smartgears.ContextProvider;
import org.gcube.smartgears.context.application.ApplicationContext;
@Singleton
public class GCubeVolatileStorageBackendFactory implements StorageBackendFactory {
private StorageBackend singleton;
private static final String PROP_PREFIX = "volatile.";
private static final String PROP_PREFIX = "volatile.";
public static final String NAME = "volatile-minio";
@PostConstruct
public void init(){
S3Backend s3Backend = new S3Backend(new PayloadBackend(getName(), getParameters()), (String) -> UUID.randomUUID().toString());
s3Backend.setPayloadConfiguration(new PayloadBackend(getName(),null));
public void init() {
S3Backend s3Backend = new S3Backend(new PayloadBackend(getName(), getParameters()),
(String) -> UUID.randomUUID().toString());
s3Backend.setPayloadConfiguration(new PayloadBackend(getName(), null));
this.singleton = s3Backend;
}
@Override
public String getName() {
return NAME;
return NAME;
}
@Override
public boolean isSystemStorage() {
return true;
}
@Override
public StorageBackend create(PayloadBackend payloadConfiguration) throws InvalidCallParameters {
if (payloadConfiguration.getParameters().isEmpty())
if (payloadConfiguration.getParameters().isEmpty())
return singleton;
throw new InvalidCallParameters("gcubeMinIo created with not empty parameters");
}
private Metadata getParameters(){
try (InputStream input = GCubeVolatileStorageBackendFactory.class.getClassLoader().getResourceAsStream("storage-settings.properties")) {
private Metadata getParameters() {
ApplicationContext context = ContextProvider.get();
String folderPath = context.appSpecificConfigurationFolder().toString();
try (InputStream input = new FileInputStream(Paths.get(folderPath, "storage-settings.properties").toFile())) {
Properties prop = new Properties();
@ -61,15 +66,17 @@ public class GCubeVolatileStorageBackendFactory implements StorageBackendFactory
Map<String, Object> params = new HashMap<String, Object>();
prop.forEach((k,v) -> { if (k.toString().startsWith(PROP_PREFIX)) params.put(k.toString().replace(PROP_PREFIX, ""), v.toString());});
prop.forEach((k, v) -> {
if (k.toString().startsWith(PROP_PREFIX))
params.put(k.toString().replace(PROP_PREFIX, ""), v.toString());
});
return new Metadata(params);
} catch (IOException ex) {
throw new RuntimeException("error initializing MinIO", ex);
}
}
}

8
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeDefaultS3StorageBackendFactory.java
View File

@ -1,7 +1,9 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@ -16,6 +18,8 @@ import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.smartgears.ContextProvider;
import org.gcube.smartgears.context.application.ApplicationContext;
@Singleton
public class GcubeDefaultS3StorageBackendFactory implements StorageBackendFactory {
@ -50,7 +54,9 @@ public class GcubeDefaultS3StorageBackendFactory implements StorageBackendFactor
}
private Metadata getParameters(){
try (InputStream input = GcubeDefaultS3StorageBackendFactory.class.getClassLoader().getResourceAsStream("storage-settings.properties")) {
ApplicationContext context = ContextProvider.get();
String folderPath = context.appSpecificConfigurationFolder().toString();
try (InputStream input = new FileInputStream(Paths.get(folderPath, "storage-settings.properties").toFile())) {
Properties prop = new Properties();

15
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java
View File

@ -18,9 +18,9 @@ import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.ObjectMetadata;
@ -52,16 +52,18 @@ public class S3Backend extends StorageBackend{
try {
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secret);
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.setProtocol(Protocol.HTTPS);
clientConfig.setProtocol(Protocol.HTTPS);
client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials)).withClientConfiguration(clientConfig).build();
.withEndpointConfiguration(new EndpointConfiguration(url,"us-east-1" ))
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.enablePathStyleAccess()
.withClientConfiguration(clientConfig).build();
client.setEndpoint(url);
client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
if (createBucket && !client.doesBucketExistV2(bucketName)) {
log.debug("creating {} bucket",this.bucketName);
client.createBucket(bucketName);
log.debug("bucket {} created",this.bucketName);
}
@ -129,9 +131,6 @@ public class S3Backend extends StorageBackend{
public MetaInfo upload(InputStream stream, String relativePath, String name, String storageId, Long size, String user) {
try {
/*Map<String, String> headers = new HashMap<>();
headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY");
Map<String, String> userMetadata = new HashMap<>();*/
ObjectMetadata objMeta = new ObjectMetadata();
objMeta.addUserMetadata("user", user);
objMeta.addUserMetadata("title", name);

5
src/test/java/org/gcube/data/access/fs/external/CreateUsers.java vendored
View File

@ -155,13 +155,14 @@ public class CreateUsers {
public void createUser2AndAddItToVres() throws Exception{
client.impersonate(user);
client.createUserAccount(user2);
/*client.getVreFolderManager(vreWA).addUser(user2);
client.getVreFolderManager(vreWA).addUser(user2);
client.getVreFolderManager(vreRO).addUser(user2);
client.getVreFolderManager(vreWO).addUser(user2);*/
client.getVreFolderManager(vreWO).addUser(user2);
}
@Test
public void printVREFoldersAdmin() throws StorageHubException{
System.out.println("launching ....." );
List<FolderContainer> vreFolders = client.getVREFolders().getContainers().stream().map(c -> ((FolderContainer) c)).collect(Collectors.toList());
for (FolderContainer fc : vreFolders) {
System.out.println(fc.get().getTitle());