gCubeSystem
/
storagehub
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

- managing of vre folder specific backend

This commit is contained in:
lucio 2023-06-01 19:00:42 +02:00
parent b34ad84baf
commit 87cc8a3ff9
19 changed files with 250 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Dockerfile-test Normal file
View File

@ -0,0 +1,12 @@
FROM smartgears-distribution:4.0.0-java11-tomcat9
ARG REPOUSER=admin
ARG REPOPWD=admin
COPY ./target/storagehub-test-storages.war /tomcat/webapps/storagehub.war
COPY ./docker/jackrabbit /app/jackrabbit
COPY ./docker/storagehub.xml /tomcat/conf/Catalina/localhost/
COPY ./docker/logback.xml /etc/
COPY ./docker/container.ini /etc/
RUN unzip /tomcat/webapps/storagehub.war -d /tomcat/webapps/storagehub
RUN rm /tomcat/webapps/storagehub.war
COPY ./docker/storage-settings.properties /tomcat/webapps/storagehub/WEB-INF/classes/
RUN sed -i "s/{{adminId}}/$REPOUSER/g; s/{{adminPwd}}/$REPOPWD/g" /tomcat/webapps/storagehub/WEB-INF/web.xml

39
docker-compose-test.yml Normal file
View File

@ -0,0 +1,39 @@
version: '3.7'
services:
postgres:
image: postgres:10.5
restart: always
environment:
- POSTGRES_DB=workspace-db
- POSTGRES_USER=ws-db-user
- POSTGRES_PASSWORD=dbPwd
logging:
options:
max-size: 10m
max-file: "3"
ports:
- '5423:5432'
volumes:
- ./postgres-data:/var/lib/postgresql/data
copy the sql script to create tables
- ./sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
storagehub:
build:
dockerfile: ./Dockerfile-test
ports:
- '8081:8080'
minio:
image: minio/minio
ports:
- "9000:9000"
- "9001:9001"
volumes:
- minio_storage:/data
environment:
MINIO_ROOT_USER: SHUBTEST
MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCY
command: server --console-address ":9001" /data
volumes:
minio_storage: {}

2
docker-compose.yml
View File

@ -17,7 +17,7 @@ services:
- ./postgres-data:/var/lib/postgresql/data
copy the sql script to create tables
- ./sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
storagehub:
storagehub:
build: .
ports:
- '8081:8080'

8
docker/storage-settings.properties
View File

@ -6,5 +6,9 @@ default.createBucket=true
volatile.bucketName=storagehub-volatile-dev
volatile.key=SHUBTEST
volatile.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
volatile.url=http://minio:9000
volatile.createBucket=true
volatile.url=http://minio:9000
volatile.createBucket=true
gcube-minio.key=SHUBTEST
gcube-minio.secret=wJalrXUtnFEMI/K7MDENG/bPxRfiCY
gcube-minio.url=http://minio:9000
gcube-minio.createBucket=true

2
enunciate.xml
View File

@ -19,6 +19,6 @@
<additional-css
file="css/d4science_enunciate_custom.css" />
</docs>
<swagger basePath="/${project.artifactId}/workspace" />
<swagger basePath="/workspace" />
</modules>
</enunciate>

29
pom.xml
View File

@ -474,4 +474,33 @@
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>integration</id>
<build>
<finalName>storagehub-test-storages</finalName>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<executions>
<execution>
<phase>process-test-classes</phase>
<configuration>
<target>
<copy todir="${basedir}/target/classes">
<fileset dir="${basedir}/target/test-classes" includes="org/gcube/data/access/storages/**/*" />
</copy>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

2
src/main/java/org/gcube/data/access/storagehub/StorageHub.java
View File

@ -15,6 +15,7 @@ import org.gcube.data.access.storagehub.services.ItemSharing;
import org.gcube.data.access.storagehub.services.ItemsCreator;
import org.gcube.data.access.storagehub.services.ItemsManager;
import org.gcube.data.access.storagehub.services.MessageManager;
import org.gcube.data.access.storagehub.services.StorageManager;
import org.gcube.data.access.storagehub.services.UserManager;
import org.gcube.data.access.storagehub.services.WorkspaceManager;
import org.gcube.data.access.storagehub.services.admin.ScriptManager;
@ -37,6 +38,7 @@ public class StorageHub extends Application {
classes.add(GroupManager.class);
classes.add(ScriptManager.class);
classes.add(MessageManager.class);
classes.add(StorageManager.class);
classes.add(DocsGenerator.class);
classes.add(MultiPartFeature.class);
classes.add(SerializableErrorEntityTextWriter.class);

7
src/main/java/org/gcube/data/access/storagehub/handlers/items/builders/FolderCreationParameters.java
View File

@ -1,10 +1,10 @@
package org.gcube.data.access.storagehub.handlers.items.builders;
import java.util.Map;
import java.util.Objects;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.plugins.PluginParameters;
public class FolderCreationParameters extends CreateParameters {
@ -82,11 +82,12 @@ public class FolderCreationParameters extends CreateParameters {
String plugin;
protected BackendCreationBuilder(String pluginName, FolderCreationBuilder cb) {
this.cb = cb;
this.plugin = pluginName;
}
public FolderCreationBuilder withParameters(PluginParameters params){
this.cb.parameters.backend = new PayloadBackend(plugin, new Metadata(params.getParameters()));
public FolderCreationBuilder withParameters(Map<String,Object> params){
this.cb.parameters.backend = new PayloadBackend(plugin, new Metadata(params));
return this.cb;
}

4
src/main/java/org/gcube/data/access/storagehub/health/DefaultStorageCheck.java
View File

@ -5,7 +5,7 @@ import org.gcube.common.health.api.ReadinessChecker;
import org.gcube.common.health.api.response.HealthCheckResponse;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.data.access.storagehub.handlers.plugins.StorageBackendHandler;
import org.gcube.data.access.storagehub.storage.backend.impl.GcubeMinIoStorageBackendFactory;
import org.gcube.data.access.storagehub.storage.backend.impl.GcubeDefaultMinIoStorageBackendFactory;
import org.gcube.data.access.storagehub.storage.backend.impl.S3Backend;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -25,7 +25,7 @@ public class DefaultStorageCheck implements HealthCheck{
@Override
public HealthCheckResponse check() {
try {
GcubeMinIoStorageBackendFactory storageFactory =new GcubeMinIoStorageBackendFactory();
GcubeDefaultMinIoStorageBackendFactory storageFactory =new GcubeDefaultMinIoStorageBackendFactory();
storageFactory.init();
if (((S3Backend)storageFactory.create(defaultPayload)).isAlive())
return HealthCheckResponse.builder(getName()).up().build();

18
src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java
View File

@ -5,6 +5,7 @@ import static org.gcube.data.access.storagehub.Roles.VREMANAGER_ROLE;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
@ -19,6 +20,7 @@ import javax.jcr.security.Privilege;
import javax.servlet.ServletContext;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@ -148,7 +150,7 @@ public class GroupManager {
@Path("")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@AuthorizationControl(allowedRoles={INFRASTRUCTURE_MANAGER_ROLE}, exception=MyAuthException.class)
public String createGroup(@FormDataParam("group") String group, @FormDataParam("accessType") AccessType accessType, @FormDataParam("folderOwner") String folderOwner){
public String createGroup(@FormDataParam("group") String group, @FormDataParam("accessType") AccessType accessType, @FormDataParam("folderOwner") String folderOwner, @FormDataParam("useDefaultStorage") @DefaultValue("true") boolean useDefaultStorage){
InnerMethodName.instance.set("createGroup");
@ -167,7 +169,7 @@ public class GroupManager {
User user = (User)usrManager.getAuthorizable(folderOwner);
createVreFolder(groupId, session, accessType!=null?accessType:AccessType.WRITE_OWNER, folderOwner);
createVreFolder(groupId, session, accessType!=null?accessType:AccessType.WRITE_OWNER, folderOwner, useDefaultStorage);
boolean success = this.internalAddUserToGroup(session, createdGroup, user);
@ -518,7 +520,7 @@ public class GroupManager {
return users;
}
private void createVreFolder(String groupId, JackrabbitSession session, AccessType defaultAccessType, String owner ) throws Exception{
private void createVreFolder(String groupId, JackrabbitSession session, AccessType defaultAccessType, String owner, boolean useDefaultStorage ) throws Exception{
Node sharedRootNode = session.getNode(Constants.SHARED_FOLDER_PATH);
@ -533,11 +535,15 @@ public class GroupManager {
String displayName = groupId.replaceAll(root+"-[^\\-]*\\-(.*)", "$1");
log.info("creating vreFolder with name {} and title {} and owner {}", name, displayName, owner);
log.info("creating vreFolder with name {} and title {} and owner {} and default storage {}", name, displayName, owner, useDefaultStorage);
FolderCreationParameters folderParameters = FolderCreationParameters.builder().name(name).description( "VREFolder for "+groupId).author(owner).on(sharedRootNode.getIdentifier()).with(session).build();
FolderCreationParameters folderParameters;
if (!useDefaultStorage)
folderParameters = FolderCreationParameters.builder().onRepository("gcube-minio").withParameters(Collections.singletonMap("bucketName", name+"-gcube-vre")).name(name).description( "VREFolder for "+groupId).author(owner).on(sharedRootNode.getIdentifier()).with(session).build();
else
folderParameters = FolderCreationParameters.builder().name(name).description( "VREFolder for "+groupId).author(owner).on(sharedRootNode.getIdentifier()).with(session).build();
Node folder= Utils.createFolderInternally(folderParameters, null, true);
Node folder= Utils.createFolderInternally(folderParameters, null, useDefaultStorage);
folder.setPrimaryType(PrimaryNodeType.NT_WORKSPACE_SHARED_FOLDER);
folder.setProperty(NodeProperty.IS_VRE_FOLDER.toString(), true);
folder.setProperty(NodeProperty.TITLE.toString(), name);

40
src/main/java/org/gcube/data/access/storagehub/services/StorageManager.java Normal file
View File

@ -0,0 +1,40 @@
package org.gcube.data.access.storagehub.services;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import org.gcube.common.storagehub.model.storages.StorageDescriptor;
import org.gcube.data.access.storagehub.StorageHubAppllicationManager;
import org.gcube.data.access.storagehub.handlers.plugins.StorageBackendHandler;
import org.gcube.smartgears.annotations.ManagedBy;
import org.gcube.smartgears.utils.InnerMethodName;
import com.webcohesion.enunciate.metadata.rs.RequestHeader;
import com.webcohesion.enunciate.metadata.rs.RequestHeaders;
@Path("storages")
@ManagedBy(StorageHubAppllicationManager.class)
@RequestHeaders({
@RequestHeader( name = "Authorization", description = "Bearer token, see https://dev.d4science.org/how-to-access-resources"),
})
public class StorageManager {
@Inject
StorageBackendHandler storageBackendHandler;
@GET
@Path("/")
@Produces(MediaType.APPLICATION_JSON)
public List<StorageDescriptor> getStorages(){
InnerMethodName.instance.set("getStorages");
List<StorageDescriptor> storages = new ArrayList<>();
storageBackendHandler.getAllImplementations().forEach( f -> storages.add(new StorageDescriptor(f.getName())));
return storages;
}
}

5
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeMongoStorageBackendFactory.java
View File

@ -16,6 +16,11 @@ public class GCubeMongoStorageBackendFactory implements StorageBackendFactory {
return Constants.MONGO_STORAGE;
}
@Override
public boolean isSystemStorage() {
return true;
}
@Override
public StorageBackend create(PayloadBackend payloadConfiguration) throws InvalidCallParameters {
return new GCubeMongoStorageBackend(payloadConfiguration);

5
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeVolatileStorageBackendFactory.java
View File

@ -39,6 +39,11 @@ public class GCubeVolatileStorageBackendFactory implements StorageBackendFactory
return NAME;
}
@Override
public boolean isSystemStorage() {
return true;
}
@Override
public StorageBackend create(PayloadBackend payloadConfiguration) throws InvalidCallParameters {
if (payloadConfiguration.getParameters().isEmpty())

9
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeMinIoStorageBackendFactory.java → src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeDefaultMinIoStorageBackendFactory.java
View File

@ -18,7 +18,7 @@ import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
@Singleton
public class GcubeMinIoStorageBackendFactory implements StorageBackendFactory {
public class GcubeDefaultMinIoStorageBackendFactory implements StorageBackendFactory {
private StorageBackend singleton;
@ -35,6 +35,11 @@ public class GcubeMinIoStorageBackendFactory implements StorageBackendFactory {
public String getName() {
return Constants.DEFAULT_MINIO_STORAGE;
}
@Override
public boolean isSystemStorage() {
return true;
}
@Override
public StorageBackend create(PayloadBackend payloadConfiguration) throws InvalidCallParameters {
@ -45,7 +50,7 @@ public class GcubeMinIoStorageBackendFactory implements StorageBackendFactory {
}
private Metadata getParameters(){
try (InputStream input = GcubeMinIoStorageBackendFactory.class.getClassLoader().getResourceAsStream("storage-settings.properties")) {
try (InputStream input = GcubeDefaultMinIoStorageBackendFactory.class.getClassLoader().getResourceAsStream("storage-settings.properties")) {
Properties prop = new Properties();

74
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeMinIOStorageBackendFactory.java Normal file
View File

@ -0,0 +1,74 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import javax.annotation.PostConstruct;
import javax.inject.Singleton;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
@Singleton
public class GcubeMinIOStorageBackendFactory implements StorageBackendFactory {
private static final String PROP_PREFIX = "gcube-minio.";
private Metadata baseParameters;
@PostConstruct
public void init(){
baseParameters = getParameters();
}
@Override
public String getName() {
return "gcube-minio";
}
@Override
public boolean isSystemStorage() {
return true;
}
@Override
public StorageBackend create(PayloadBackend payloadConfiguration) throws InvalidCallParameters {
if (payloadConfiguration.getParameters().isEmpty())
throw new InvalidCallParameters(getName()+": null or empty parameter not allowed");
String bucketName = (String)payloadConfiguration.getParameters().get("bucketName");
if (bucketName == null || bucketName.isBlank())
throw new InvalidCallParameters(getName()+": 'bucketName' cannot be blank or empty");
Metadata metadata = new Metadata(new HashMap<>(baseParameters.getMap()));
metadata.getMap().putAll(payloadConfiguration.getParameters());
return new S3Backend(new PayloadBackend(getName(), metadata), (String) -> UUID.randomUUID().toString());
}
private Metadata getParameters(){
try (InputStream input = GcubeDefaultMinIoStorageBackendFactory.class.getClassLoader().getResourceAsStream("storage-settings.properties")) {
Properties prop = new Properties();
prop.load(input);
Map<String, Object> params = new HashMap<String, Object>();
prop.forEach((k,v) -> { if (k.toString().startsWith(PROP_PREFIX)) params.put(k.toString().replace(PROP_PREFIX, ""), v);});
return new Metadata(params);
} catch (IOException ex) {
throw new RuntimeException("error initializing MinIO", ex);
}
}
}

6
src/main/resources/storage-settings.properties
View File

@ -7,4 +7,8 @@ volatile.bucketName=shub-volatile-dev
volatile.key=qa4zD0QqmcG0JZNG
volatile.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL
volatile.url=https://minio.d4science.org/
volatile.createBucket=false
volatile.createBucket=false
gcube-minio.key=qa4zD0QqmcG0JZNG
gcube-minio.secret=9jzHtOigmrprBYtPGIkh3Tq1Bago4zxL
gcube-minio.url=https://minio.d4science.org/
gcube-minio.createBucket=true

2
src/test/java/org/gcube/data/access/fs/external/CreateUsers.java vendored
View File

@ -136,7 +136,7 @@ public class CreateUsers {
@Test
public void addUserToVREs() throws Exception{
client.getVreFolderManager(vreWA).createVRE(AccessType.WRITE_ALL, user);
client.getVreFolderManager(vreWA).createVRE(AccessType.WRITE_ALL, user, false);
client.getVreFolderManager(vreWA).setAdmin(user);
client.getVreFolderManager(vreRO).createVRE(AccessType.READ_ONLY, user);
client.getVreFolderManager(vreRO).setAdmin(user);

4
src/test/java/org/gcube/data/access/storages/mock/MockStorage.java → src/test/java/org/gcube/data/access/storages/mock/FSStorage.java
View File

@ -17,9 +17,9 @@ import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
public class MockStorage extends StorageBackend {
public class FSStorage extends StorageBackend {
public MockStorage(PayloadBackend payloadConfiguration) {
public FSStorage(PayloadBackend payloadConfiguration) {
super(payloadConfiguration);
}

6
src/test/java/org/gcube/data/access/storages/mock/MockStorageFactory.java → src/test/java/org/gcube/data/access/storages/mock/FSStorageFactory.java
View File

@ -8,16 +8,16 @@ import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
@Singleton
public class MockStorageFactory implements StorageBackendFactory {
public class FSStorageFactory implements StorageBackendFactory {
@Override
public String getName() {
return "MockStorage";
return "local-filesystem";
}
@Override
public StorageBackend create(PayloadBackend payloadConfiguration) throws InvalidCallParameters {
return new MockStorage(payloadConfiguration);
return new FSStorage(payloadConfiguration);
}
}