package org.gcube.data.access.storagehub; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URL; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Deque; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.function.Predicate; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import javax.jcr.Node; import javax.jcr.NodeIterator; import javax.jcr.PathNotFoundException; import javax.jcr.RepositoryException; import javax.jcr.Session; import javax.jcr.lock.Lock; import javax.jcr.lock.LockException; import javax.jcr.nodetype.NodeType; import javax.jcr.query.Query; import javax.jcr.version.Version; import org.apache.commons.io.FilenameUtils; import org.apache.jackrabbit.util.ISO9075; import org.apache.jackrabbit.util.Text; import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.common.storagehub.model.Excludes; import org.gcube.common.storagehub.model.Paths; import org.gcube.common.storagehub.model.exceptions.BackendGenericError; import org.gcube.common.storagehub.model.exceptions.ItemLockedException; import org.gcube.common.storagehub.model.exceptions.UserNotAuthorizedException; import org.gcube.common.storagehub.model.items.AbstractFileItem; import org.gcube.common.storagehub.model.items.ExternalLink; import org.gcube.common.storagehub.model.items.FolderItem; import org.gcube.common.storagehub.model.items.GCubeItem; import org.gcube.common.storagehub.model.items.Item; import org.gcube.common.storagehub.model.items.SharedFolder; import org.gcube.common.storagehub.model.types.ItemAction; import org.gcube.common.storagehub.model.types.NodeProperty; import org.gcube.data.access.storagehub.accounting.AccountingHandler; import org.gcube.data.access.storagehub.handlers.StorageBackendHandler; import org.gcube.data.access.storagehub.handlers.VRE; import org.gcube.data.access.storagehub.handlers.VREManager; import org.gcube.data.access.storagehub.handlers.VersionHandler; import org.gcube.data.access.storagehub.handlers.items.Item2NodeConverter; import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import net.bull.javamelody.internal.common.LOG; public class Utils { public final static String SERVICE_NAME = "home-library"; public final static String SERVICE_CLASS = "org.gcube.portlets.user"; private static final String FOLDERS_TYPE = "nthl:workspaceItem"; private static final Logger logger = LoggerFactory.getLogger(Utils.class); public static String getSecurePassword(String user) throws Exception { String digest = null; try { MessageDigest md = MessageDigest.getInstance("MD5"); byte[] hash = md.digest(user.getBytes("UTF-8")); //converting byte array to Hexadecimal String StringBuilder sb = new StringBuilder(2*hash.length); for(byte b : hash){ sb.append(String.format("%02x", b&0xff)); } digest = sb.toString(); } catch (Exception e) { logger.error("error getting secure password",e); } return digest; } public static long getItemCount(Node parent, boolean showHidden, Class nodeType) throws RepositoryException, BackendGenericError{ return getItemList(parent, Excludes.ALL, null, showHidden, nodeType).size(); } public static void acquireLockWithWait(Session ses, String nodePath, boolean isDeep, String login, int maxTries) throws RepositoryException, ItemLockedException { Lock lock = null; int tries = 0; while(lock==null && tries<=maxTries) { try { lock = ses.getWorkspace().getLockManager().lock(nodePath, isDeep, true, 0,login); logger.info("lock acquired(try n. {})", tries); }catch (LockException e) { try { if (maxTries>=tries) { int waitTime = (int)(Math.random()*5000); logger.info("lock NOT acquired, waiting (try n. {}) for {}", tries, waitTime); Thread.sleep(waitTime); } } catch (InterruptedException e1) {} tries++; } } if (lock==null) throw new ItemLockedException("the item is locked"); } public static List serachByNameOnFolder(Session ses, AuthorizationChecker authChecker, Node parent, List excludes, Range range, boolean showHidden, boolean excludeTrashed, Class nodeTypeToInclude, String nameParam) throws RepositoryException, BackendGenericError{ String xpath = String.format("/jcr:root%s//element(*,nthl:workspaceItem)[jcr:like(fn:lower-case(@jcr:title), '%s')]",ISO9075.encodePath(parent.getPath()), nameParam.toLowerCase()); //String query = String.format("SELECT * FROM [nthl:workspaceLeafItem] AS node WHERE ISDESCENDANTNODE('%s') ORDER BY node.[jcr:lastModified] DESC ",vreFolder.getPath()); logger.debug("query for search is {}",xpath); long start = System.currentTimeMillis(); Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(xpath, Query.XPATH); NodeChildrenFilterIterator iterator = new NodeChildrenFilterIterator(jcrQuery.execute().getNodes()); logger.debug("[SEARCH] real search took {} millis",(System.currentTimeMillis()-start)); Predicate checker = new Predicate() { @Override public boolean test(Node t) { try { authChecker.checkReadAuthorizationControl(t.getSession(), t.getIdentifier()); return true; } catch (UserNotAuthorizedException | BackendGenericError | RepositoryException e) { return false; } } }; return getItemListFromNodeIterator(checker, iterator , excludes, range, showHidden, excludeTrashed, nodeTypeToInclude); } public static List getItemList(Node parent, List excludes, Range range, boolean showHidden, Class nodeTypeToInclude) throws RepositoryException, BackendGenericError{ return getItemList(null, parent, excludes, range, showHidden, nodeTypeToInclude); } public static List getItemList(Predicate checker, Node parent, List excludes, Range range, boolean showHidden, Class nodeTypeToInclude) throws RepositoryException, BackendGenericError{ logger.trace("getting children of node {}", parent.getIdentifier()); long start = System.currentTimeMillis(); NodeChildrenFilterIterator iterator = new NodeChildrenFilterIterator(parent); logger.trace("time to get iterator {}",(System.currentTimeMillis()-start)); return getItemListFromNodeIterator(checker, iterator, excludes, range, showHidden, false, nodeTypeToInclude); } private static List getItemListFromNodeIterator(Predicate checker, NodeChildrenFilterIterator iterator, List excludes, Range range, boolean showHidden, boolean excludeTrashed, Class nodeTypeToInclude) throws RepositoryException, BackendGenericError{ List returnList = new ArrayList(); logger.trace("nodeType is {}",nodeTypeToInclude); int count =0; logger.trace("selected range is {}", range); Node2ItemConverter node2Item= new Node2ItemConverter(); Set duplicateId = new HashSet(); while (iterator.hasNext()){ Node current = iterator.next(); logger.debug("[SEARCH] evaluating node {} ",current.hasProperty(NodeProperty.TITLE.toString())? current.getProperty(NodeProperty.TITLE.toString()):current.getName()); //REMOVE duplicate nodes, in case the indexes are not working if (duplicateId.contains(current.getIdentifier())) { logger.warn("duplicated node found"); continue; } //EXCLUDES node from predicate if (checker!=null && !checker.test(current)) continue; if (isToExclude(current, showHidden)) continue; logger.debug("[SEARCH] current node not excluded {} ",current.hasProperty(NodeProperty.TITLE.toString())? current.getProperty(NodeProperty.TITLE.toString()):current.getName()); if (range==null || (count>=range.getStart() && returnList.size() getAllNodesForZip(FolderItem directory, Session session, AccountingHandler accountingHandler, List excludes) throws RepositoryException, BackendGenericError{ Deque queue = new LinkedList(); Node currentNode = session.getNodeByIdentifier(directory.getId()); queue.push(directory); Deque tempQueue = new LinkedList(); logger.trace("adding directory {}",currentNode.getPath()); for (Item item : Utils.getItemList(currentNode,Excludes.GET_ONLY_CONTENT, null, false, null)){ if (excludes.contains(item.getId())) continue; if (item instanceof FolderItem) tempQueue.addAll(getAllNodesForZip((FolderItem) item, session, accountingHandler, excludes)); else if (item instanceof AbstractFileItem){ logger.trace("adding file {}",item.getPath()); AbstractFileItem fileItem = (AbstractFileItem) item; accountingHandler.createReadObj(fileItem.getTitle(), session, session.getNodeByIdentifier(item.getId()), false); queue.addLast(item); } } queue.addAll(tempQueue); return queue; } public static void zipNode(ZipOutputStream zos, Deque queue, String login, org.gcube.common.storagehub.model.Path originalPath, StorageBackendHandler storageHandler) throws Exception{ logger.trace("originalPath is {}",originalPath.toPath()); org.gcube.common.storagehub.model.Path actualPath = Paths.getPath(""); while (!queue.isEmpty()) { Item item = queue.pop(); if (item instanceof FolderItem) { actualPath = Paths.getPath(item.getPath()); logger.trace("actualPath is {}",actualPath.toPath()); String name = Paths.remove(actualPath, originalPath).toPath().replaceFirst("/", ""); logger.trace("writing dir {}",name); if (name.isEmpty()) continue; try { zos.putNextEntry(new ZipEntry(name)); }finally { zos.closeEntry(); } } else if (item instanceof AbstractFileItem){ try { InputStream streamToWrite = storageHandler.download(((AbstractFileItem)item).getContent().getStorageId()); if (streamToWrite == null){ logger.warn("discarding item {} ",item.getName()); continue; } try(BufferedInputStream is = new BufferedInputStream(streamToWrite)){ String name = (Paths.remove(actualPath, originalPath).toPath()+item.getName()).replaceFirst("/", ""); logger.trace("writing file {}",name); zos.putNextEntry(new ZipEntry(name)); copyStream(is, zos); }catch (Exception e) { logger.warn("error writing item {}", item.getName(),e); } finally{ zos.closeEntry(); } zos.flush(); }catch (Throwable e) { logger.warn("error reading content for item {}", item.getPath(),e); } } } zos.close(); } private static void copyStream(InputStream in, OutputStream out) throws IOException { byte[] buffer = new byte[2048]; int readcount = 0; while ((readcount=in.read(buffer))!=-1) { out.write(buffer, 0, readcount); } } public static boolean hasSharedChildren(Node node) throws RepositoryException, BackendGenericError{ Node2ItemConverter node2Item = new Node2ItemConverter(); NodeChildrenFilterIterator children = new NodeChildrenFilterIterator(node); while (children.hasNext()) { Node child= children.next(); if (node2Item.checkNodeType(child, SharedFolder.class)) return true; if (node2Item.checkNodeType(child, FolderItem.class) && hasSharedChildren(child)) return true; } return false; } public static void getAllContentIds(Session ses, Set idsToDelete, Item itemToDelete, VersionHandler versionHandler) throws Exception{ if (itemToDelete instanceof AbstractFileItem) { Node currentNode = ses.getNodeByIdentifier(itemToDelete.getId()); List ntList = Arrays.asList(currentNode.getMixinNodeTypes()); boolean isVersioned = false; logger.debug("mixin node type are {}",ntList); for (NodeType nt: ntList) if(nt.getName().equals("mix:versionable")) { isVersioned = true; break; } if (isVersioned) { List versions = versionHandler.getContentVersionHistory(currentNode, ses); versions.forEach(v -> { try { String storageId =v.getFrozenNode().getProperty(NodeProperty.STORAGE_ID.toString()).getString(); idsToDelete.add(storageId); logger.info("retrieved StorageId {} for version {}", storageId, v.getName()); } catch (Exception e) { logger.warn("error retreiving sotrageId version for item with id {}",itemToDelete.getId(),e); } }); } else logger.info("not versionable node type found of type {}", currentNode.getPrimaryNodeType().toString()); idsToDelete.add(((AbstractFileItem) itemToDelete).getContent().getStorageId()); }else if (itemToDelete instanceof FolderItem) { List items = Utils.getItemList(ses.getNodeByIdentifier(itemToDelete.getId()), Excludes.GET_ONLY_CONTENT , null, true, null); for (Item item: items) getAllContentIds(ses, idsToDelete, item, versionHandler); } } public static String checkExistanceAndGetUniqueName(Session ses, Node destination, String name) throws BackendGenericError{ try { destination.getNode(name); }catch(PathNotFoundException pnf) { return Text.escapeIllegalJcrChars(name); } catch (Exception e) { throw new BackendGenericError(e); } try { String filename = FilenameUtils.getBaseName(name); String ext = FilenameUtils.getExtension(name); String nameTocheck = ext.isEmpty()? String.format("%s(*)",filename): String.format("%s(*).%s",filename, ext); logger.trace("filename is {}, extension is {} , and name to check is {}", filename, ext, nameTocheck); NodeIterator ni = destination.getNodes(nameTocheck); int maxval = 0; while (ni.hasNext()) { Node n = ni.nextNode(); int actual = Integer.parseInt(n.getName().replaceAll(String.format("%s\\((\\d*)\\).*", filename), "$1")); if (actual>maxval) maxval = actual; } String newName = ext.isEmpty()? String.format("%s(%d)", filename,maxval+1) : String.format("%s(%d).%s", filename,maxval+1, ext) ; return Text.escapeIllegalJcrChars(newName); } catch (Exception e) { throw new BackendGenericError(e); } } public static Node createFolderInternally(Session ses, Node destinationNode, String name, String description, boolean hidden, String login, AccountingHandler accountingHandler) throws BackendGenericError { logger.debug("creating folder {} in {}", name, destinationNode); String uniqueName = Utils.checkExistanceAndGetUniqueName(ses, destinationNode, name); FolderItem item = new FolderItem(); Calendar now = Calendar.getInstance(); item.setName(uniqueName); item.setTitle(uniqueName); item.setDescription(description); //item.setCreationTime(now); boolean hiddenDestNode= false; try { hiddenDestNode = destinationNode.getProperty(NodeProperty.HIDDEN.toString()).getBoolean(); }catch (Throwable e) {} item.setHidden(hidden || hiddenDestNode); item.setLastAction(ItemAction.CREATED); item.setLastModificationTime(now); item.setLastModifiedBy(login); item.setOwner(login); item.setPublicItem(false); Node newNode = new Item2NodeConverter().getNode(destinationNode, item); if (accountingHandler!=null) { accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, destinationNode, false); accountingHandler.createEntryCreate(item.getTitle(), ses, newNode, false); } return newNode; } public static Node createURLInternally(Session ses, Node destinationNode, String name, URL value, String description, String login, AccountingHandler accountingHandler) throws BackendGenericError { String uniqueName = Utils.checkExistanceAndGetUniqueName(ses, destinationNode, name); ExternalLink item = new ExternalLink(); Calendar now = Calendar.getInstance(); item.setName(uniqueName); item.setTitle(uniqueName); item.setDescription(description); //item.setCreationTime(now); item.setLastAction(ItemAction.CREATED); item.setLastModificationTime(now); item.setLastModifiedBy(login); item.setOwner(login); item.setPublicItem(false); item.setValue(value); try { item.setHidden(destinationNode.getProperty(NodeProperty.HIDDEN.toString()).getBoolean()); } catch (Throwable e) { item.setHidden(false); } Node newNode = new Item2NodeConverter().getNode(destinationNode, item); if (accountingHandler!=null) { accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, destinationNode, false); accountingHandler.createEntryCreate(item.getTitle(), ses, newNode, false); } return newNode; } public static Node createGcubeItemInternally(Session ses, Node destinationNode, String name, String description, String login, GCubeItem gcubeItem, AccountingHandler accountingHandler) throws BackendGenericError { Calendar now = Calendar.getInstance(); gcubeItem.setName(name); gcubeItem.setTitle(name); gcubeItem.setDescription(description); //item.setCreationTime(now); gcubeItem.setHidden(false); gcubeItem.setLastAction(ItemAction.CREATED); gcubeItem.setLastModificationTime(now); gcubeItem.setLastModifiedBy(login); gcubeItem.setOwner(login); //to inherit hidden property //item.setHidden(destinationItem.isHidden()); Node newNode = new Item2NodeConverter().getNode(destinationNode, gcubeItem); //TODO: accounting for GCUBEITEM //accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, newNode, false); return newNode; } public static void setPropertyOnChangeNode(Node node, String login, ItemAction action) throws RepositoryException { node.setProperty(NodeProperty.LAST_MODIFIED.toString(), Calendar.getInstance()); node.setProperty(NodeProperty.LAST_MODIFIED_BY.toString(), login); node.setProperty(NodeProperty.LAST_ACTION.toString(), action.name()); } public static synchronized VRE getVreFolderItem(Session ses, Node2ItemConverter node2Item, VREManager vreManager, List excludes ) throws RepositoryException, BackendGenericError{ org.gcube.common.storagehub.model.Path vrePath = Paths.append(Utils.getWorkspacePath(), Constants.VRE_FOLDER_PARENT_NAME); ScopeBean bean = new ScopeBean(ScopeProvider.instance.get()); if (!bean.is(Type.VRE)) throw new BackendGenericError("the current scope is not a VRE"); String entireScopeName= bean.toString().replaceAll("^/(.*)/?$", "$1").replaceAll("/", "-"); VRE vre = vreManager.getVRE(entireScopeName); if (vre!=null) return vre; else { String query = String.format("SELECT * FROM [nthl:workspaceItem] As node WHERE node.[jcr:title] like '%s' AND ISDESCENDANTNODE('%s')",entireScopeName, vrePath.toPath()); Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(query, Constants.QUERY_LANGUAGE); NodeIterator it = jcrQuery.execute().getNodes(); if (!it.hasNext()) throw new BackendGenericError("vre folder not found for context "+entireScopeName); Node folder = it.nextNode(); Item vreFolder = node2Item.getItem(folder, excludes); return vreManager.putVRE(vreFolder); } } public static synchronized VRE getVreFolderItemByGroupNameAndUser(Session ses, String goupName, String userId, Node2ItemConverter node2Item, VREManager vreManager, List excludes ) throws RepositoryException, BackendGenericError{ org.gcube.common.storagehub.model.Path vrePath = Paths.append(Utils.getWorkspacePath(userId), Constants.VRE_FOLDER_PARENT_NAME); VRE vre = vreManager.getVRE(goupName); if (vre!=null) return vre; else { String query = String.format("SELECT * FROM [nthl:workspaceItem] As node WHERE node.[jcr:title] like '%s' AND ISDESCENDANTNODE('%s')",goupName, vrePath.toPath()); Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(query, Constants.QUERY_LANGUAGE); NodeIterator it = jcrQuery.execute().getNodes(); if (!it.hasNext()) throw new BackendGenericError("vre folder not found for context "+goupName); Node folder = it.nextNode(); Item vreFolder = node2Item.getItem(folder, excludes); return vreManager.putVRE(vreFolder); } } }