This commit is contained in:
Lucio Lelii 2019-07-29 14:26:08 +00:00
parent 1c1e067a19
commit b5b9a862e0
19 changed files with 257 additions and 32 deletions

View File

@ -4,7 +4,7 @@
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
<dependent-module archiveName="storagehub-model-1.0.4-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/storagehub-model_TRUNK/storagehub-model_TRUNK">
<dependent-module archiveName="storagehub-model-1.0.5-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/storagehub-model_TRUNK/storagehub-model_TRUNK">
<dependency-type>uses</dependency-type>
</dependent-module>
<property name="context-root" value="storagehub"/>

17
pom.xml
View File

@ -23,7 +23,7 @@
<jackrabbit.version>2.16.0</jackrabbit.version>
<tomcat.version>7.0.40</tomcat.version>
<jetty.version>6.1.26</jetty.version>
<tika.version>1.7</tika.version>
<tika.version>1.21</tika.version>
<slf4j.api.version>1.6.6</slf4j.api.version>
<slf4j.version>1.7.4</slf4j.version> <!-- sync with logback version -->
<logback.version>1.0.12</logback.version>
@ -149,6 +149,21 @@
<version>${jackrabbit.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.tika/tika-parsers -->
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parsers</artifactId>
<version>1.21</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.tika/tika-core -->
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>1.21</version>
</dependency>
<!-- jersey -->
<dependency>
<groupId>javax.ws.rs</groupId>

View File

@ -4,6 +4,7 @@ import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Calendar;
@ -31,6 +32,7 @@ import org.gcube.common.storagehub.model.Paths;
import org.gcube.common.storagehub.model.exceptions.BackendGenericError;
import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.ExternalLink;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.GCubeItem;
import org.gcube.common.storagehub.model.items.Item;
@ -325,6 +327,32 @@ public class Utils {
accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, newNode, false);
return newNode;
}
public static Node createURLInternally(Session ses, Node destinationNode, String name, URL value, String description, String login, AccountingHandler accountingHandler) throws BackendGenericError {
String uniqueName = Utils.checkExistanceAndGetUniqueName(ses, destinationNode, name);
ExternalLink item = new ExternalLink();
Calendar now = Calendar.getInstance();
item.setName(uniqueName);
item.setTitle(uniqueName);
item.setDescription(description);
//item.setCreationTime(now);
item.setLastAction(ItemAction.CREATED);
item.setLastModificationTime(now);
item.setLastModifiedBy(login);
item.setOwner(login);
item.setPublicItem(false);
item.setValue(value);
//to inherit hidden property
//item.setHidden(destinationItem.isHidden());
Node newNode = new Item2NodeConverter().getNode(destinationNode, item);
if (accountingHandler!=null)
accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, newNode, false);
return newNode;
}
public static Node createGcubeItemInternally(Session ses, Node destinationNode, String name, String description, String login, GCubeItem gcubeItem, AccountingHandler accountingHandler) throws BackendGenericError {

View File

@ -1,6 +1,7 @@
package org.gcube.data.access.storagehub.handlers;
import java.lang.reflect.Field;
import java.net.URL;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashSet;
@ -149,6 +150,7 @@ public class Item2NodeConverter {
if (returnType.equals(String.class)) return new Values(new StringValue((String) value));
if (returnType.isEnum()) return new Values(new StringValue(((Enum) value).toString()));
if (returnType.equals(Calendar.class)) return new Values(new DateValue((Calendar) value));
if (returnType.equals(URL.class)) return new Values(new StringValue(((URL) value).toString()));
if (returnType.equals(Boolean.class) || returnType.equals(boolean.class)) return new Values(new BooleanValue((Boolean) value));
if (returnType.equals(Long.class) || returnType.equals(long.class)) return new Values(new LongValue((Long) value));
if (returnType.equals(Integer.class) || returnType.equals(int.class)) return new Values(new LongValue((Long) value));
@ -224,19 +226,25 @@ public class Item2NodeConverter {
Node metadataNode;
try {
metadataNode = node.getNode(NodeProperty.METADATA.toString());
metadataNode.remove();
}catch (PathNotFoundException e) {
metadataNode = node.addNode(NodeProperty.METADATA.toString());
}
metadataNode = node.addNode(NodeProperty.METADATA.toString());
for (Field field : retrieveAllFields(Metadata.class)){
if (field.isAnnotationPresent(MapAttribute.class)){
//logger.debug("found field {} of type annotated as MapAttribute in class {}", field.getName(), clazz.getName());
field.setAccessible(true);
for (Entry<String, Object> entry : meta.entrySet())
try{
Values values = getObjectValue(entry.getValue().getClass(), entry.getValue());
if (values.isMulti()) metadataNode.setProperty(entry.getKey(), values.getValues());
else metadataNode.setProperty(entry.getKey(), values.getValue());
if (entry.getValue() != null) {
Values values = getObjectValue(entry.getValue().getClass(), entry.getValue());
if (values.isMulti()) metadataNode.setProperty(entry.getKey(), values.getValues());
else metadataNode.setProperty(entry.getKey(), values.getValue());
}
} catch (Exception e ) {
logger.warn("error setting value",e);
}

View File

@ -1,6 +1,8 @@
package org.gcube.data.access.storagehub.handlers;
import java.lang.reflect.Field;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
@ -255,6 +257,7 @@ public class Node2ItemConverter {
if (returnType.equals(String.class)) return prop.getString();
if (returnType.isEnum()) return Enum.valueOf(returnType, prop.getString());
if (returnType.equals(Calendar.class)) return prop.getDate();
if (returnType.equals(URL.class)) return new URI(prop.getString()).toURL();
if (returnType.equals(Boolean.class) || returnType.equals(boolean.class)) return prop.getBoolean();
if (returnType.equals(Long.class) || returnType.equals(long.class)) return prop.getLong();
if (returnType.equals(Integer.class) || returnType.equals(int.class)) return prop.getLong();

View File

@ -40,4 +40,12 @@ public class StorageBackendHandler {
defaultBackend.delete(id);
}
public String getTotalVolume() {
return defaultBackend.getTotalSizeStored();
}
public String getTotalItemsCount() {
return defaultBackend.getTotalItemsCount();
}
}

View File

@ -7,11 +7,10 @@ import org.gcube.common.storagehub.model.items.nodes.Content;
public interface ContentHandler {
void initiliseSpecificContent(InputStream is, String fileName) throws Exception;
void initiliseSpecificContent(InputStream is, String fileName, String mimeType) throws Exception;
Content getContent();
AbstractFileItem buildItem(String name, String description, String login);
}

View File

@ -11,8 +11,11 @@ public class GenericFileHandler implements ContentHandler{
Content content = new Content();
@Override
public void initiliseSpecificContent(InputStream is, String filename) throws Exception {}
public void initiliseSpecificContent(InputStream is, String filename, String mimeType) throws Exception {
content.setMimeType(mimeType);
}
@Override
public Content getContent() {

View File

@ -29,7 +29,7 @@ public class ImageHandler implements ContentHandler{
private static final Logger logger = LoggerFactory.getLogger(ImageHandler.class);
@Override
public void initiliseSpecificContent(InputStream is, String fileName) throws Exception {
public void initiliseSpecificContent(InputStream is, String fileName, String mimeType) throws Exception {
Image image = javax.imageio.ImageIO.read(is);
int width = image.getWidth(null);
@ -50,6 +50,7 @@ public class ImageHandler implements ContentHandler{
}catch(Throwable t) {
logger.warn("thumbnail for file {} cannot be created ", fileName,t);
}
content.setMimeType(mimeType);
}

View File

@ -0,0 +1,57 @@
package org.gcube.data.access.storagehub.handlers.content;
import java.io.InputStream;
import java.util.Calendar;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;
import org.apache.tika.sax.BodyContentHandler;
import org.gcube.common.storagehub.model.annotations.MimeTypeHandler;
import org.gcube.common.storagehub.model.items.GenericFileItem;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.types.ItemAction;
public class OfficeAppHandler implements ContentHandler{
Content content = new Content();
@Override
public void initiliseSpecificContent(InputStream is, String filename, String mimeType) throws Exception {
//detecting the file type
BodyContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
ParseContext pcontext = new ParseContext();
//OOXml parser
OOXMLParser msofficeparser = new OOXMLParser ();
msofficeparser.parse(is, handler, metadata,pcontext);
String appname = metadata.get("Application-Name");
content.setMimeType("application/"+appname);
}
@Override
public Content getContent() {
return content;
}
@Override
public GenericFileItem buildItem(String name, String description, String login) {
GenericFileItem item = new GenericFileItem();
Calendar now = Calendar.getInstance();
item.setName(name);
item.setTitle(name);
item.setDescription(description);
//item.setCreationTime(now);
item.setHidden(false);
item.setLastAction(ItemAction.CREATED);
item.setLastModificationTime(now);
item.setLastModifiedBy(login);
item.setOwner(login);
item.setContent(this.content);
return item;
}
}

View File

@ -21,10 +21,10 @@ public class PdfHandler implements ContentHandler {
public static final String TITLE = "dc:title";
PDFContent content = new PDFContent();
@Override
public void initiliseSpecificContent(InputStream is, String fileName) throws Exception {
public void initiliseSpecificContent(InputStream is, String fileName, String mimeType) throws Exception {
PdfReader reader = new PdfReader(is);
content.setNumberOfPages(Long.valueOf(reader.getNumberOfPages()));
content.setVersion(String.valueOf(reader.getPdfVersion()));
@ -32,6 +32,7 @@ public class PdfHandler implements ContentHandler {
content.setAuthor(fileInfo.containsKey(AUTHOR)?fileInfo.get(AUTHOR):"n/a");
content.setProducer(fileInfo.containsKey(PRODUCER)?fileInfo.get(PRODUCER):"n/a");
content.setTitle(fileInfo.containsKey(TITLE)?fileInfo.get(TITLE):"n/a");
content.setMimeType(mimeType);
}
@Override

View File

@ -14,7 +14,6 @@ import javax.jcr.security.AccessControlManager;
import javax.jcr.security.Privilege;
import javax.servlet.ServletContext;
import javax.ws.rs.Consumes;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
@ -24,7 +23,10 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.jackrabbit.api.JackrabbitSession;
import org.apache.jackrabbit.api.security.JackrabbitAccessControlList;
import org.apache.jackrabbit.api.security.user.Group;
import org.apache.jackrabbit.api.security.user.User;
import org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils;
import org.gcube.common.gxrest.response.outbound.GXOutboundErrorResponse;
import org.gcube.common.storagehub.model.Excludes;
@ -38,6 +40,7 @@ import org.gcube.common.storagehub.model.exceptions.UserNotAuthorizedException;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.SharedFolder;
import org.gcube.common.storagehub.model.items.VreFolder;
import org.gcube.common.storagehub.model.types.ACLList;
import org.gcube.data.access.storagehub.AuthorizationChecker;
import org.gcube.data.access.storagehub.handlers.CredentialHandler;
@ -152,13 +155,28 @@ public class ACLManager {
if (accessType!=AccessType.ADMINISTRATOR)
throw new InvalidCallParameters("acls in vreFolder cannot be changed, only new admin can be set");
org.apache.jackrabbit.api.security.user.UserManager usrManager = ((JackrabbitSession)ses).getUserManager();
String groupId = folder.getTitle();
Group group = (Group)usrManager.getAuthorizable(groupId);
User authUser = (User)usrManager.getAuthorizable(user);
//TODO if is a VRE FOLDER
if (!group.isMember(authUser))
throw new InvalidCallParameters("user "+user+" is not in the group "+groupId);
String path = node.getPath();
AccessControlManager acm = ses.getAccessControlManager();
JackrabbitAccessControlList acls = AccessControlUtils.getAccessControlList(acm, path);
Privilege[] userPrivileges = new Privilege[] { acm.privilegeFromName(AccessType.ADMINISTRATOR.getValue()) };
Principal principal = AccessControlUtils.getPrincipal(ses, user);
acls.addAccessControlEntry(principal, userPrivileges);
acm.setPolicy(path, acls);
ses.save();
} else {
//TODO: probably must be changed with the shared set
if (!((SharedFolder) item).getUsers().getMap().containsKey(user))
throw new InvalidCallParameters("shared folder with id "+item.getId()+" is not shared with user "+user);

View File

@ -30,7 +30,6 @@ import org.apache.jackrabbit.api.security.user.Query;
import org.apache.jackrabbit.api.security.user.QueryBuilder;
import org.apache.jackrabbit.api.security.user.User;
import org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils;
import org.gcube.common.authorization.control.annotations.AuthorizationControl;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.gxrest.response.outbound.GXOutboundErrorResponse;
import org.gcube.common.storagehub.model.acls.AccessType;
@ -39,7 +38,6 @@ import org.gcube.common.storagehub.model.types.NodeProperty;
import org.gcube.common.storagehub.model.types.PrimaryNodeType;
import org.gcube.data.access.storagehub.Constants;
import org.gcube.data.access.storagehub.Utils;
import org.gcube.data.access.storagehub.exception.MyAuthException;
import org.gcube.data.access.storagehub.handlers.CredentialHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -57,7 +55,7 @@ public class GroupManager {
@GET
@Path("")
@Produces(MediaType.APPLICATION_JSON)
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public List<String> getGroups(){
JackrabbitSession session = null;
@ -91,7 +89,7 @@ public class GroupManager {
@POST
@Path("")
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public String createGroup(@FormParam("group") String group, @FormParam("accessType") AccessType accessType){
JackrabbitSession session = null;
@ -120,7 +118,7 @@ public class GroupManager {
@DELETE
@Path("{group}")
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public String deleteGroup(@PathParam("group") String group){
JackrabbitSession session = null;
@ -153,7 +151,7 @@ public class GroupManager {
@PUT
@Path("{id}")
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public boolean addUserToGroup(@PathParam("id") String groupId, @FormParam("userId") String userId){
JackrabbitSession session = null;
@ -190,7 +188,7 @@ public class GroupManager {
@DELETE
@Path("{groupId}/users/{userId}")
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public boolean removeUserFromGroup(@PathParam("groupId") String groupId, @PathParam("userId") String userId){
JackrabbitSession session = null;
@ -234,7 +232,7 @@ public class GroupManager {
@GET
@Path("{groupId}/users")
@Produces(MediaType.APPLICATION_JSON)
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public List<String> getUsersOfGroup(@PathParam("groupId") String groupId){
JackrabbitSession session = null;

View File

@ -3,6 +3,7 @@ package org.gcube.data.access.storagehub.services;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
@ -154,6 +155,60 @@ public class ItemsCreator {
return Response.ok(toReturn).build();
}
@POST
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Path("/{id}/create/URL")
public Response createURL(@PathParam("id") String id, @FormParam("name") String name, @FormParam("description") String description, @FormParam("value") URL value) {
InnerMethodName.instance.set("createItem(URL)");
log.info("create url called");
Session ses = null;
String toReturn = null;
try{
final String login = AuthorizationProvider.instance.get().getClient().getId();
long start = System.currentTimeMillis();
ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context));
log.info("time to connect to repo {}",(System.currentTimeMillis()-start));
Node destination;
try {
destination = ses.getNodeByIdentifier(id);
}catch(RepositoryException inf) {
throw new IdNotFoundException(id);
}
if (!node2Item.checkNodeType(destination, FolderItem.class))
throw new InvalidItemException("the destination item is not a folder");
authChecker.checkWriteAuthorizationControl(ses, destination.getIdentifier(), true);
Utils.acquireLockWithWait(ses, destination.getPath(), false, login, 10);
Node newNode;
try {
newNode = Utils.createURLInternally(ses, destination, name, value, description, login, accountingHandler);
ses.save();
} finally {
ses.getWorkspace().getLockManager().unlock(destination.getPath());
}
log.info("item with id {} correctly created",newNode.getIdentifier());
toReturn = newNode.getIdentifier();
}catch(StorageHubException she ){
log.error(she.getErrorMessage(), she);
GXOutboundErrorResponse.throwException(she, Response.Status.fromStatusCode(she.getStatus()));
}catch(RepositoryException re ){
log.error("jcr error creating item", re);
GXOutboundErrorResponse.throwException(new BackendGenericError("jcr error creating item", re));
}finally{
if (ses!=null)
ses.logout();
}
return Response.ok(toReturn).build();
}
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Path("/{id}/create/GCUBEITEM")
@ -452,8 +507,8 @@ public class ItemsCreator {
handler = contenthandlerFactory.create(mimeType);
is1.reset();
handler.initiliseSpecificContent(is1, name);
handler.getContent().setMimeType(mimeType);
handler.initiliseSpecificContent(is1, name, mimeType);
log.trace("TIMING: reading the mimetype - finished in {}",System.currentTimeMillis()-start);
} catch (Throwable e) {
log.error("error retrieving mimeType",e);

View File

@ -55,7 +55,7 @@ public class UserManager {
@GET
@Path("")
@Produces(MediaType.APPLICATION_JSON)
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public List<String> getUsers(){
JackrabbitSession session = null;
@ -89,7 +89,7 @@ public class UserManager {
@POST
@Path("")
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public String createUser(@FormParam("user") String user, @FormParam("password") String password){
JackrabbitSession session = null;
@ -127,7 +127,7 @@ public class UserManager {
@DELETE
@Path("{id}")
@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
//@AuthorizationControl(allowed={"lucio.lelii"}, exception=MyAuthException.class)
public String deleteUser(@PathParam("id") String id){
JackrabbitSession session = null;

View File

@ -49,6 +49,7 @@ import org.gcube.data.access.storagehub.Utils;
import org.gcube.data.access.storagehub.handlers.CredentialHandler;
import org.gcube.data.access.storagehub.handlers.Item2NodeConverter;
import org.gcube.data.access.storagehub.handlers.Node2ItemConverter;
import org.gcube.data.access.storagehub.handlers.StorageBackendHandler;
import org.gcube.data.access.storagehub.handlers.TrashHandler;
import org.gcube.data.access.storagehub.handlers.VRE;
import org.gcube.data.access.storagehub.handlers.VREManager;
@ -91,6 +92,8 @@ public class WorkspaceManager {
@Inject Node2ItemConverter node2Item;
@Inject Item2NodeConverter item2Node;
@Inject StorageBackendHandler storageBackend;
@Path("")
@GET
@Produces(MediaType.APPLICATION_JSON)
@ -416,5 +419,21 @@ public class WorkspaceManager {
return new ItemList(toReturn);
}
@Path("count")
@GET
public String getTotalItemsCount(){
InnerMethodName.instance.set("getTotalItemsCount");
return storageBackend.getTotalItemsCount();
}
@Path("size")
@GET
public String getTotalVolume(){
InnerMethodName.instance.set("getTotalSize");
return storageBackend.getTotalVolume();
}
}

View File

@ -69,6 +69,18 @@ public class GCubeStorageBackend implements StorageBackend {
}
@Override
public String getTotalSizeStored() {
IClient storageClient = getStorageClient(AuthorizationProvider.instance.get().getClient().getId()).getClient();
return storageClient.getUserTotalItems();
}
@Override
public String getTotalItemsCount() {
IClient storageClient = getStorageClient(AuthorizationProvider.instance.get().getClient().getId()).getClient();
return storageClient.getTotalUserVolume();
}

View File

@ -25,7 +25,7 @@ no. 654119), SoBigData (grant no. 654024), AGINFRA PLUS (grant no. 731001).
Version
--------------------------------------------------
1.0.7-SNAPSHOT (2019-07-03)
1.0.7-SNAPSHOT (2019-07-29)
Please see the file named "changelog.xml" in this directory for the release notes.

View File

@ -10,7 +10,7 @@
</context-param>
<context-param>
<param-name>resolver-basepath</param-name>
<param-value>https://data-pre.d4science.org/shub</param-value>
<param-value>https://data-d.d4science.org/shub</param-value>
</context-param>
<servlet>
<servlet-name>org.gcube.data.access.storagehub.StorageHub</servlet-name>