gCubeSystem
/
storagehub
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

upload by url added

This commit is contained in:
lucio.lelii 2021-10-15 19:51:57 +02:00
parent da7385f62f
commit 9facccdf46
5 changed files with 169 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [v1.5.0-SNAPSHOT] - [2021-10-15]
## [v1.4.0] - [2021-10-07] ## [v1.4.0] - [2021-10-07]
- slow query removed from VRE retrieving and recents - slow query removed from VRE retrieving and recents

9
src/main/java/org/gcube/data/access/storagehub/MultipleOutputStream.java
View File

@ -25,7 +25,7 @@ public class MultipleOutputStream {
this.is = is; this.is = is;
logger.debug("requested {} piped streams ",number); logger.debug("UPLOAD: requested {} piped streams ",number);
pipedInStreams = new MyPipedInputStream[number]; pipedInStreams = new MyPipedInputStream[number];
pipedOutStreams = new MyPipedOututStream[number]; pipedOutStreams = new MyPipedOututStream[number];
@ -39,6 +39,8 @@ public class MultipleOutputStream {
public void startWriting() throws IOException{ public void startWriting() throws IOException{
logger.debug("UPLOAD: started writing multiple streams");
BufferedInputStream bis = new BufferedInputStream(is); BufferedInputStream bis = new BufferedInputStream(is);
byte[] buf = new byte[1024*64]; byte[] buf = new byte[1024*64];
int read=-1; int read=-1;
@ -49,7 +51,8 @@ public class MultipleOutputStream {
pipedOutStreams[i].write(buf, 0, read); pipedOutStreams[i].write(buf, 0, read);
} }
} }
logger.trace("UPLOAD: written byte "+writeTot);
writeTot+= read; writeTot+= read;
if (allOutStreamClosed()) if (allOutStreamClosed())
@ -64,7 +67,7 @@ public class MultipleOutputStream {
} }
} }
logger.debug("total written {} ",writeTot); logger.debug("UPLOAD: total written {} ",writeTot);
} }

20
src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java
View File

@ -247,7 +247,9 @@ public class ItemHandler {
} }
private Node createFileItemInternally(Session ses, Node destinationNode, InputStream stream, String name, String description, String login, boolean withLock) throws RepositoryException, StorageHubException{ private Node createFileItemInternally(Session ses, Node destinationNode, InputStream stream, String name, String description, String login, boolean withLock) throws RepositoryException, StorageHubException{
log.debug("UPLOAD: starting preparing file");
Node newNode; Node newNode;
FolderItem destinationItem = node2Item.getItem(destinationNode, Excludes.ALL); FolderItem destinationItem = node2Item.getItem(destinationNode, Excludes.ALL);
FolderManager folderManager = pluginHandler.getFolderManager(destinationItem); FolderManager folderManager = pluginHandler.getFolderManager(destinationItem);
@ -314,14 +316,16 @@ public class ItemHandler {
private AbstractFileItem fillItemWithContent(InputStream stream, StorageBackend storageBackend, String name, String description, String relPath, String login) throws BackendGenericError{ private AbstractFileItem fillItemWithContent(InputStream stream, StorageBackend storageBackend, String name, String description, String relPath, String login) throws BackendGenericError{
log.debug("UPLOAD: filling content");
ContentHandler handler = getContentHandler(stream, storageBackend, name, relPath, login); ContentHandler handler = getContentHandler(stream, storageBackend, name, relPath, login);
AbstractFileItem item =handler.buildItem(name, description, login); AbstractFileItem item =handler.buildItem(name, description, login);
return item ; return item ;
} }
private ContentHandler getContentHandler(InputStream stream, StorageBackend storageBackend, String name, String relPath, String login) throws BackendGenericError { private ContentHandler getContentHandler(InputStream stream, StorageBackend storageBackend, String name, String relPath, String login) throws BackendGenericError {
log.debug("UPLOAD: handling content");
final MultipleOutputStream mos; final MultipleOutputStream mos;
try{ try{
mos = new MultipleOutputStream(stream, 2); mos = new MultipleOutputStream(stream, 2);
@ -351,7 +355,7 @@ public class ItemHandler {
is1.reset(); is1.reset();
handler.initiliseSpecificContent(is1, name, mimeType); handler.initiliseSpecificContent(is1, name, mimeType);
log.trace("TIMING: reading the mimetype - finished in {}",System.currentTimeMillis()-start); log.debug("UPLOAD: reading the mimetype - finished in {}",System.currentTimeMillis()-start);
} catch (Throwable e) { } catch (Throwable e) {
log.error("error retrieving mimeType",e); log.error("error retrieving mimeType",e);
throw new RuntimeException(e); throw new RuntimeException(e);
@ -366,7 +370,9 @@ public class ItemHandler {
@Override @Override
public MetaInfo call() throws Exception { public MetaInfo call() throws Exception {
try(InputStream is1 = mos.get()){ try(InputStream is1 = mos.get()){
log.debug("UPLOAD: upload on storage - start");
MetaInfo info = storageBackend.upload(is1, relPath, name); MetaInfo info = storageBackend.upload(is1, relPath, name);
log.debug("UPLOAD: upload on storage - stop");
return info; return info;
}catch (Throwable e) { }catch (Throwable e) {
log.error("error writing content",e ); log.error("error writing content",e );
@ -380,13 +386,13 @@ public class ItemHandler {
Future<MetaInfo> uploaderF = executor.submit(AuthorizedTasks.bind(uploader)); Future<MetaInfo> uploaderF = executor.submit(AuthorizedTasks.bind(uploader));
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
log.debug("TIMING: writing the stream - start"); log.debug("UPLOAD: writing the stream - start");
try { try {
mos.startWriting(); mos.startWriting();
log.debug("TIMING: writing the stream - finished in {}",System.currentTimeMillis()-start);
ContentHandler handler = detectorF.get(); ContentHandler handler = detectorF.get();
MetaInfo info = uploaderF.get(); MetaInfo info = uploaderF.get();
log.debug("UPLOAD: writing the stream - finished in {}",System.currentTimeMillis()-start);
handler.getContent().setData(NodeConstants.CONTENT_NAME); handler.getContent().setData(NodeConstants.CONTENT_NAME);
handler.getContent().setStorageId(info.getStorageId()); handler.getContent().setStorageId(info.getStorageId());
handler.getContent().setSize(info.getSize()); handler.getContent().setSize(info.getSize());

162
src/main/java/org/gcube/data/access/storagehub/services/ItemsCreator.java
View File

@ -2,17 +2,20 @@ package org.gcube.data.access.storagehub.services;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI;
import java.net.URL; import java.net.URL;
import javax.inject.Inject; import javax.inject.Inject;
import javax.jcr.RepositoryException; import javax.jcr.RepositoryException;
import javax.jcr.Session; import javax.jcr.Session;
import javax.servlet.ServletContext; import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes; import javax.ws.rs.Consumes;
import javax.ws.rs.FormParam; import javax.ws.rs.FormParam;
import javax.ws.rs.POST; import javax.ws.rs.POST;
import javax.ws.rs.Path; import javax.ws.rs.Path;
import javax.ws.rs.PathParam; import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context; import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
@ -44,12 +47,12 @@ import org.slf4j.LoggerFactory;
public class ItemsCreator extends Impersonable{ public class ItemsCreator extends Impersonable{
private static final Logger log = LoggerFactory.getLogger(ItemsCreator.class); private static final Logger log = LoggerFactory.getLogger(ItemsCreator.class);
@Context @Context
ServletContext context; ServletContext context;
RepositoryInitializer repository = StorageHubAppllicationManager.repository; RepositoryInitializer repository = StorageHubAppllicationManager.repository;
@Inject @Inject
ItemHandler itemHandler; ItemHandler itemHandler;
@ -116,7 +119,7 @@ public class ItemsCreator extends Impersonable{
} }
return Response.ok(toReturn).build(); return Response.ok(toReturn).build();
} }
@POST @POST
@Consumes(MediaType.APPLICATION_FORM_URLENCODED) @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Path("/{id}/create/URL") @Path("/{id}/create/URL")
@ -127,7 +130,7 @@ public class ItemsCreator extends Impersonable{
String toReturn = null; String toReturn = null;
try{ try{
ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context)); ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context));
ItemsParameterBuilder<URLCreationParameters> builder = URLCreationParameters.builder().name(name).description(description).url(value).on(id).with(ses).author(currentUser); ItemsParameterBuilder<URLCreationParameters> builder = URLCreationParameters.builder().name(name).description(description).url(value).on(id).with(ses).author(currentUser);
toReturn = itemHandler.create(builder.build()); toReturn = itemHandler.create(builder.build());
@ -147,8 +150,8 @@ public class ItemsCreator extends Impersonable{
} }
return Response.ok(toReturn).build(); return Response.ok(toReturn).build();
} }
@POST @POST
@Consumes(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON)
@Path("/{id}/create/GCUBEITEM") @Path("/{id}/create/GCUBEITEM")
@ -161,7 +164,7 @@ public class ItemsCreator extends Impersonable{
try{ try{
ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context)); ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context));
ItemsParameterBuilder<GCubeItemCreationParameters> builder = GCubeItemCreationParameters.builder().item(item).on(id).with(ses).author(currentUser); ItemsParameterBuilder<GCubeItemCreationParameters> builder = GCubeItemCreationParameters.builder().item(item).on(id).with(ses).author(currentUser);
toReturn = itemHandler.create(builder.build()); toReturn = itemHandler.create(builder.build());
}catch(StorageHubException she ){ }catch(StorageHubException she ){
log.error(she.getErrorMessage(), she); log.error(she.getErrorMessage(), she);
@ -181,24 +184,26 @@ public class ItemsCreator extends Impersonable{
@POST @POST
@Consumes(MediaType.MULTIPART_FORM_DATA) @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Path("/{id}/create/FILE") @Path("/{id}/create/FILE")
public String createFileItem(@PathParam("id") String id, @FormDataParam("name") String name, public String createFileItemFromUrl(@PathParam("id") String id, @FormParam("name") String name,
@FormDataParam("description") String description, @FormParam("description") String description,
@FormDataParam("file") InputStream stream, @FormParam("url") String url){
@FormDataParam("file") FormDataContentDisposition fileDetail){ InnerMethodName.instance.set("createItem(FILEFromUrl)");
InnerMethodName.instance.set("createItem(FILE)");
Session ses = null; Session ses = null;
String toReturn = null; String toReturn = null;
try{ try{
log.debug("UPLOAD: call started");
ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context)); ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context));
ItemsParameterBuilder<FileCreationParameters> builder = FileCreationParameters.builder().name(name).description(description).stream(stream).fileDetails(fileDetail) try(InputStream stream = new URI(url).toURL().openStream()){
.on(id).with(ses).author(currentUser); ItemsParameterBuilder<FileCreationParameters> builder = FileCreationParameters.builder().name(name).description(description).stream(stream)
.on(id).with(ses).author(currentUser);
toReturn = itemHandler.create(builder.build()); log.debug("UPLOAD: item prepared");
toReturn = itemHandler.create(builder.build());
}
log.debug("UPLOAD: call finished");
}catch(RepositoryException re ){ }catch(RepositoryException re ){
log.error("jcr error creating file item", re); log.error("jcr error creating file item", re);
GXOutboundErrorResponse.throwException(new BackendGenericError("jcr error creating file item", re)); GXOutboundErrorResponse.throwException(new BackendGenericError("jcr error creating file item", re));
@ -218,9 +223,118 @@ public class ItemsCreator extends Impersonable{
} }
@POST
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
@Path("/{id}/create/FILE")
public String createFileItemStream(@PathParam("id") String id, @QueryParam("name") String name,
@QueryParam("description") String description, @Context HttpServletRequest request){
InnerMethodName.instance.set("createItem(FILEByStream)");
Session ses = null;
String toReturn = null;
try{
log.debug("UPLOAD: call started");
ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context));
ItemsParameterBuilder<FileCreationParameters> builder = FileCreationParameters.builder().name(name).description(description).stream(request.getInputStream())
.on(id).with(ses).author(currentUser);
log.debug("UPLOAD: item prepared");
toReturn = itemHandler.create(builder.build());
log.debug("UPLOAD: call finished");
}catch(RepositoryException re ){
log.error("jcr error creating file item", re);
GXOutboundErrorResponse.throwException(new BackendGenericError("jcr error creating file item", re));
}catch(StorageHubException she ){
log.error(she.getErrorMessage(), she);
GXOutboundErrorResponse.throwException(she, Response.Status.fromStatusCode(she.getStatus()));
}catch(Throwable e ){
log.error("unexpected error", e);
GXOutboundErrorResponse.throwException(new BackendGenericError(e));
}finally{
if (ses!=null && ses.isLive()) {
log.info("session closed");
ses.logout();
}
}
return toReturn;
}
@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Path("/{id}/create/FILE")
public String createFileItem(@PathParam("id") String id, @FormDataParam("name") String name,
@FormDataParam("description") String description,
@FormDataParam("file") InputStream stream,
@FormDataParam("file") FormDataContentDisposition fileDetail){
InnerMethodName.instance.set("createItem(FILE)");
Session ses = null;
String toReturn = null;
try{
log.debug("UPLOAD: call started");
ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context));
ItemsParameterBuilder<FileCreationParameters> builder = FileCreationParameters.builder().name(name).description(description).stream(stream).fileDetails(fileDetail)
.on(id).with(ses).author(currentUser);
log.debug("UPLOAD: item prepared");
toReturn = itemHandler.create(builder.build());
log.debug("UPLOAD: call finished");
}catch(RepositoryException re ){
log.error("jcr error creating file item", re);
GXOutboundErrorResponse.throwException(new BackendGenericError("jcr error creating file item", re));
}catch(StorageHubException she ){
log.error(she.getErrorMessage(), she);
GXOutboundErrorResponse.throwException(she, Response.Status.fromStatusCode(she.getStatus()));
}catch(Throwable e ){
log.error("unexpected error", e);
GXOutboundErrorResponse.throwException(new BackendGenericError(e));
}finally{
if (ses!=null && ses.isLive()) {
log.info("session closed");
ses.logout();
}
}
return toReturn;
}
@POST
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Path("/{id}/create/ARCHIVE")
public String uploadArchiveFromURL(@PathParam("id") String id, @FormParam("parentFolderName") String parentFolderName,
@FormParam("url") String url){
InnerMethodName.instance.set("createItem(ARCHIVEFromURL)");
Session ses = null;
String toReturn = null;
try{
ses = repository.getRepository().login(CredentialHandler.getAdminCredentials(context));
try(InputStream stream = new URI(url).toURL().openStream()){
ItemsParameterBuilder<ArchiveStructureCreationParameter> builder = ArchiveStructureCreationParameter.builder().parentName(parentFolderName).stream(stream)
.on(id).with(ses).author(currentUser);
toReturn = itemHandler.create(builder.build());
}
}catch(RepositoryException | ArchiveException | IOException re){
log.error("jcr error extracting archive", re);
GXOutboundErrorResponse.throwException(new BackendGenericError("jcr error extracting archive", re));
}catch(StorageHubException she ){
log.error(she.getErrorMessage(), she);
GXOutboundErrorResponse.throwException(she, Response.Status.fromStatusCode(she.getStatus()));
}catch(Throwable e ){
log.error("unexpected error", e);
GXOutboundErrorResponse.throwException(new BackendGenericError(e));
} finally{
if (ses!=null)
ses.logout();
}
return toReturn;
}
@POST @POST
@ -238,7 +352,7 @@ public class ItemsCreator extends Impersonable{
ItemsParameterBuilder<ArchiveStructureCreationParameter> builder = ArchiveStructureCreationParameter.builder().parentName(parentFolderName).stream(stream).fileDetails(fileDetail) ItemsParameterBuilder<ArchiveStructureCreationParameter> builder = ArchiveStructureCreationParameter.builder().parentName(parentFolderName).stream(stream).fileDetails(fileDetail)
.on(id).with(ses).author(currentUser); .on(id).with(ses).author(currentUser);
toReturn = itemHandler.create(builder.build()); toReturn = itemHandler.create(builder.build());
}catch(RepositoryException | ArchiveException | IOException re){ }catch(RepositoryException | ArchiveException | IOException re){
@ -259,6 +373,6 @@ public class ItemsCreator extends Impersonable{
} }
} }

14
src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java
View File

@ -128,9 +128,10 @@ public class MessageManager extends Impersonable{
Message message = node2Item.getMessageItem(messageNode); Message message = node2Item.getMessageItem(messageNode);
Node personalNode = checkRights(currentUser, message); Node personalNode = checkRights(currentUser, message);
if (countSharedSet(messageNode)>1) if (countSharedSet(messageNode)>1) {
personalNode.removeShare(); log.debug("removing node message "+personalNode.getPath());
else { personalNode.remove();
}else {
if (message.isWithAttachments()) { if (message.isWithAttachments()) {
Node attachmentNode = messageNode.getNode(Constants.ATTACHMENTNODE_NAME); Node attachmentNode = messageNode.getNode(Constants.ATTACHMENTNODE_NAME);
List<Item> attachments = Utils.getItemList(attachmentNode, Excludes.GET_ONLY_CONTENT, null, true, AbstractFileItem.class); List<Item> attachments = Utils.getItemList(attachmentNode, Excludes.GET_ONLY_CONTENT, null, true, AbstractFileItem.class);
@ -139,6 +140,7 @@ public class MessageManager extends Impersonable{
messageNode.removeSharedSet(); messageNode.removeSharedSet();
} }
ses.save(); ses.save();
log.debug("removing node message saved");
}catch (ItemNotFoundException e) { }catch (ItemNotFoundException e) {
log.error("id {} not found",id,e); log.error("id {} not found",id,e);
GXOutboundErrorResponse.throwException(new IdNotFoundException(id, e), Status.NOT_FOUND); GXOutboundErrorResponse.throwException(new IdNotFoundException(id, e), Status.NOT_FOUND);
@ -345,7 +347,7 @@ public class MessageManager extends Impersonable{
NodeIterator nodeIt = node.getNodes(); NodeIterator nodeIt = node.getNodes();
while(nodeIt.hasNext()) { while(nodeIt.hasNext()) {
Node child = nodeIt.nextNode(); Node child = nodeIt.nextNode();
log.info("message type "+child.getPrimaryNodeType().getName()); log.trace("message type "+child.getPrimaryNodeType().getName());
Message message = node2Item.getMessageItem(child); Message message = node2Item.getMessageItem(child);
if (message == null) { if (message == null) {
log.info("message discarded"); log.info("message discarded");
@ -375,13 +377,13 @@ public class MessageManager extends Impersonable{
private Node checkRights(String user, Message messageItem) throws RepositoryException, StorageHubException{ private Node checkRights(String user, Message messageItem) throws RepositoryException, StorageHubException{
Node personalNode = null; Node personalNode = null;
Node messageNode = (Node) messageItem.getRelatedNode(); Node messageNode = (Node) messageItem.getRelatedNode();
if (messageNode.getPath().startsWith(pathUtil.getWorkspacePath(currentUser).toPath())) if (messageNode.getPath().startsWith(pathUtil.getInboxPath(user).toPath()))
return messageNode; return messageNode;
NodeIterator nodeIt = messageNode.getSharedSet(); NodeIterator nodeIt = messageNode.getSharedSet();
while (nodeIt.hasNext()) { while (nodeIt.hasNext()) {
Node node = nodeIt.nextNode(); Node node = nodeIt.nextNode();
if (node.getPath().startsWith(pathUtil.getWorkspacePath(currentUser).toPath())) if (node.getPath().startsWith(pathUtil.getInboxPath(user).toPath()))
personalNode = node; personalNode = node;
} }
if (personalNode == null && if (personalNode == null &&