You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
storagehub/src/main/java/org/gcube/data/access/storagehub/Utils.java

508 lines
21 KiB
Java

package org.gcube.data.access.storagehub;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Deque;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.PathNotFoundException;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.lock.Lock;
import javax.jcr.lock.LockException;
import javax.jcr.nodetype.NodeType;
import javax.jcr.query.Query;
import javax.jcr.version.Version;
import org.apache.commons.io.FilenameUtils;
import org.apache.jackrabbit.util.ISO9075;
import org.apache.jackrabbit.util.Text;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.common.storagehub.model.Excludes;
import org.gcube.common.storagehub.model.Paths;
import org.gcube.common.storagehub.model.exceptions.BackendGenericError;
import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
import org.gcube.common.storagehub.model.exceptions.UserNotAuthorizedException;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.ExternalLink;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.GCubeItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.SharedFolder;
import org.gcube.common.storagehub.model.types.ItemAction;
import org.gcube.common.storagehub.model.types.NodeProperty;
import org.gcube.data.access.storagehub.accounting.AccountingHandler;
import org.gcube.data.access.storagehub.handlers.StorageBackendHandler;
import org.gcube.data.access.storagehub.handlers.VRE;
import org.gcube.data.access.storagehub.handlers.VREManager;
import org.gcube.data.access.storagehub.handlers.VersionHandler;
import org.gcube.data.access.storagehub.handlers.items.Item2NodeConverter;
import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.bull.javamelody.internal.common.LOG;
public class Utils {
public final static String SERVICE_NAME = "home-library";
public final static String SERVICE_CLASS = "org.gcube.portlets.user";
private static final String FOLDERS_TYPE = "nthl:workspaceItem";
private static final Logger logger = LoggerFactory.getLogger(Utils.class);
public static String getSecurePassword(String user) throws Exception {
String digest = null;
try {
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] hash = md.digest(user.getBytes("UTF-8"));
//converting byte array to Hexadecimal String
StringBuilder sb = new StringBuilder(2*hash.length);
for(byte b : hash){
sb.append(String.format("%02x", b&0xff));
}
digest = sb.toString();
} catch (Exception e) {
logger.error("error getting secure password",e);
}
return digest;
}
public static long getItemCount(Node parent, boolean showHidden, Class<? extends Item> nodeType) throws RepositoryException, BackendGenericError{
return getItemList(parent, Excludes.ALL, null, showHidden, nodeType).size();
}
public static void acquireLockWithWait(Session ses, String nodePath, boolean isDeep, String login, int maxTries) throws RepositoryException, ItemLockedException {
Lock lock = null;
int tries = 0;
while(lock==null && tries<=maxTries) {
try {
lock = ses.getWorkspace().getLockManager().lock(nodePath, isDeep, true, 0,login);
logger.info("lock acquired(try n. {})", tries);
}catch (LockException e) {
try {
if (maxTries>=tries) {
int waitTime = (int)(Math.random()*5000);
logger.info("lock NOT acquired, waiting (try n. {}) for {}", tries, waitTime);
Thread.sleep(waitTime);
}
} catch (InterruptedException e1) {}
tries++;
}
}
if (lock==null) throw new ItemLockedException("the item is locked");
}
public static <T extends Item> List<T> serachByNameOnFolder(Session ses, AuthorizationChecker authChecker, Node parent, List<String> excludes, Range range, boolean showHidden, boolean excludeTrashed, Class<? extends Item> nodeTypeToInclude, String nameParam) throws RepositoryException, BackendGenericError{
String xpath = String.format("/jcr:root%s//element(*,nthl:workspaceItem)[jcr:like(fn:lower-case(@jcr:title), '%s')]",ISO9075.encodePath(parent.getPath()), nameParam.toLowerCase());
//String query = String.format("SELECT * FROM [nthl:workspaceLeafItem] AS node WHERE ISDESCENDANTNODE('%s') ORDER BY node.[jcr:lastModified] DESC ",vreFolder.getPath());
logger.debug("query for search is {}",xpath);
long start = System.currentTimeMillis();
Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(xpath, Query.XPATH);
NodeChildrenFilterIterator iterator = new NodeChildrenFilterIterator(jcrQuery.execute().getNodes());
logger.debug("[SEARCH] real search took {} millis",(System.currentTimeMillis()-start));
Predicate<Node> checker = new Predicate<Node>() {
@Override
public boolean test(Node t) {
try {
authChecker.checkReadAuthorizationControl(t.getSession(), t.getIdentifier());
return true;
} catch (UserNotAuthorizedException | BackendGenericError | RepositoryException e) {
return false;
}
}
};
return getItemListFromNodeIterator(checker, iterator , excludes, range, showHidden, excludeTrashed, nodeTypeToInclude);
}
public static <T extends Item> List<T> getItemList(Node parent, List<String> excludes, Range range, boolean showHidden, Class<? extends Item> nodeTypeToInclude) throws RepositoryException, BackendGenericError{
return getItemList(null, parent, excludes, range, showHidden, nodeTypeToInclude);
}
public static <T extends Item> List<T> getItemList(Predicate<Node> checker, Node parent, List<String> excludes, Range range, boolean showHidden, Class<? extends Item> nodeTypeToInclude) throws RepositoryException, BackendGenericError{
logger.trace("getting children of node {}", parent.getIdentifier());
long start = System.currentTimeMillis();
NodeChildrenFilterIterator iterator = new NodeChildrenFilterIterator(parent);
logger.trace("time to get iterator {}",(System.currentTimeMillis()-start));
return getItemListFromNodeIterator(checker, iterator, excludes, range, showHidden, false, nodeTypeToInclude);
}
private static <T extends Item> List<T> getItemListFromNodeIterator(Predicate<Node> checker, NodeChildrenFilterIterator iterator, List<String> excludes, Range range, boolean showHidden, boolean excludeTrashed, Class<? extends Item> nodeTypeToInclude) throws RepositoryException, BackendGenericError{
List<T> returnList = new ArrayList<T>();
logger.trace("nodeType is {}",nodeTypeToInclude);
int count =0;
logger.trace("selected range is {}", range);
Node2ItemConverter node2Item= new Node2ItemConverter();
Set<String> duplicateId = new HashSet<String>();
while (iterator.hasNext()){
Node current = iterator.next();
logger.debug("[SEARCH] evaluating node {} ",current.hasProperty(NodeProperty.TITLE.toString())? current.getProperty(NodeProperty.TITLE.toString()):current.getName());
//REMOVE duplicate nodes, in case the indexes are not working
if (duplicateId.contains(current.getIdentifier())) {
logger.warn("duplicated node found");
continue;
}
//EXCLUDES node from predicate
if (checker!=null && !checker.test(current))
continue;
if (isToExclude(current, showHidden))
continue;
logger.debug("[SEARCH] current node not excluded {} ",current.hasProperty(NodeProperty.TITLE.toString())? current.getProperty(NodeProperty.TITLE.toString()):current.getName());
if (range==null || (count>=range.getStart() && returnList.size()<range.getLimit())) {
T item = node2Item.getFilteredItem(current, excludes, nodeTypeToInclude);
if (item==null || (item.isTrashed() && excludeTrashed)) continue;
returnList.add(item);
}
count++;
duplicateId.add(current.getIdentifier());
}
return returnList;
}
private static boolean isToExclude(Node node, boolean showHidden) throws RepositoryException{
return ((node.getName().startsWith("rep:") || (node.getName().startsWith("hl:"))) ||
(!showHidden && node.hasProperty(NodeProperty.HIDDEN.toString()) && node.getProperty(NodeProperty.HIDDEN.toString()).getBoolean()) ||
(node.getPrimaryNodeType().getName().equals(FOLDERS_TYPE) && Constants.FOLDERS_TO_EXLUDE.contains(node.getName())));
}
public static org.gcube.common.storagehub.model.Path getWorkspacePath(){
return Paths.getPath(String.format("/Home/%s/%s",AuthorizationProvider.instance.get().getClient().getId(), Constants.WORKSPACE_ROOT_FOLDER_NAME));
}
public static org.gcube.common.storagehub.model.Path getWorkspacePath(String login){
return Paths.getPath(String.format("/Home/%s/%s",login,Constants.WORKSPACE_ROOT_FOLDER_NAME));
}
public static org.gcube.common.storagehub.model.Path getHome(String login){
return Paths.getPath(String.format("/Home/%s",login));
}
public static Deque<Item> getAllNodesForZip(FolderItem directory, Session session, AccountingHandler accountingHandler, List<String> excludes) throws RepositoryException, BackendGenericError{
Deque<Item> queue = new LinkedList<Item>();
Node currentNode = session.getNodeByIdentifier(directory.getId());
queue.push(directory);
Deque<Item> tempQueue = new LinkedList<Item>();
logger.trace("adding directory {}",currentNode.getPath());
for (Item item : Utils.getItemList(currentNode,Excludes.GET_ONLY_CONTENT, null, false, null)){
if (excludes.contains(item.getId())) continue;
if (item instanceof FolderItem)
tempQueue.addAll(getAllNodesForZip((FolderItem) item, session, accountingHandler, excludes));
else if (item instanceof AbstractFileItem){
logger.trace("adding file {}",item.getPath());
AbstractFileItem fileItem = (AbstractFileItem) item;
accountingHandler.createReadObj(fileItem.getTitle(), session, session.getNodeByIdentifier(item.getId()), false);
queue.addLast(item);
}
}
queue.addAll(tempQueue);
return queue;
}
public static void zipNode(ZipOutputStream zos, Deque<Item> queue, String login, org.gcube.common.storagehub.model.Path originalPath, StorageBackendHandler storageHandler) throws Exception{
logger.trace("originalPath is {}",originalPath.toPath());
org.gcube.common.storagehub.model.Path actualPath = Paths.getPath("");
while (!queue.isEmpty()) {
Item item = queue.pop();
if (item instanceof FolderItem) {
actualPath = Paths.getPath(item.getPath());
logger.trace("actualPath is {}",actualPath.toPath());
String name = Paths.remove(actualPath, originalPath).toPath().replaceFirst("/", "");
logger.trace("writing dir {}",name);
if (name.isEmpty()) continue;
try {
zos.putNextEntry(new ZipEntry(name));
}finally {
zos.closeEntry();
}
} else if (item instanceof AbstractFileItem){
try {
InputStream streamToWrite = storageHandler.download(((AbstractFileItem)item).getContent().getStorageId());
if (streamToWrite == null){
logger.warn("discarding item {} ",item.getName());
continue;
}
try(BufferedInputStream is = new BufferedInputStream(streamToWrite)){
String name = (Paths.remove(actualPath, originalPath).toPath()+item.getName()).replaceFirst("/", "");
logger.trace("writing file {}",name);
zos.putNextEntry(new ZipEntry(name));
copyStream(is, zos);
}catch (Exception e) {
logger.warn("error writing item {}", item.getName(),e);
} finally{
zos.closeEntry();
}
zos.flush();
}catch (Throwable e) {
logger.warn("error reading content for item {}", item.getPath(),e);
}
}
}
zos.close();
}
private static void copyStream(InputStream in, OutputStream out) throws IOException {
byte[] buffer = new byte[2048];
int readcount = 0;
while ((readcount=in.read(buffer))!=-1) {
out.write(buffer, 0, readcount);
}
}
public static boolean hasSharedChildren(Node node) throws RepositoryException, BackendGenericError{
Node2ItemConverter node2Item = new Node2ItemConverter();
NodeChildrenFilterIterator children = new NodeChildrenFilterIterator(node);
while (children.hasNext()) {
Node child= children.next();
if (node2Item.checkNodeType(child, SharedFolder.class)) return true;
if (node2Item.checkNodeType(child, FolderItem.class) && hasSharedChildren(child)) return true;
}
return false;
}
public static void getAllContentIds(Session ses, Set<String> idsToDelete, Item itemToDelete, VersionHandler versionHandler) throws Exception{
if (itemToDelete instanceof AbstractFileItem) {
Node currentNode = ses.getNodeByIdentifier(itemToDelete.getId());
List<NodeType> ntList = Arrays.asList(currentNode.getMixinNodeTypes());
boolean isVersioned = false;
logger.debug("mixin node type are {}",ntList);
for (NodeType nt: ntList)
if(nt.getName().equals("mix:versionable")) {
isVersioned = true;
break;
}
if (isVersioned) {
List<Version> versions = versionHandler.getContentVersionHistory(currentNode, ses);
versions.forEach(v -> {
try {
String storageId =v.getFrozenNode().getProperty(NodeProperty.STORAGE_ID.toString()).getString();
idsToDelete.add(storageId);
logger.info("retrieved StorageId {} for version {}", storageId, v.getName());
} catch (Exception e) {
logger.warn("error retreiving sotrageId version for item with id {}",itemToDelete.getId(),e);
}
});
} else
logger.info("not versionable node type found of type {}", currentNode.getPrimaryNodeType().toString());
idsToDelete.add(((AbstractFileItem) itemToDelete).getContent().getStorageId());
}else if (itemToDelete instanceof FolderItem) {
List<Item> items = Utils.getItemList(ses.getNodeByIdentifier(itemToDelete.getId()), Excludes.GET_ONLY_CONTENT , null, true, null);
for (Item item: items)
getAllContentIds(ses, idsToDelete, item, versionHandler);
}
}
public static String checkExistanceAndGetUniqueName(Session ses, Node destination, String name) throws BackendGenericError{
try {
destination.getNode(name);
}catch(PathNotFoundException pnf) {
return Text.escapeIllegalJcrChars(name);
} catch (Exception e) {
throw new BackendGenericError(e);
}
try {
String filename = FilenameUtils.getBaseName(name);
String ext = FilenameUtils.getExtension(name);
String nameTocheck = ext.isEmpty()? String.format("%s(*)",filename): String.format("%s(*).%s",filename, ext);
logger.trace("filename is {}, extension is {} , and name to check is {}", filename, ext, nameTocheck);
NodeIterator ni = destination.getNodes(nameTocheck);
int maxval = 0;
while (ni.hasNext()) {
Node n = ni.nextNode();
int actual = Integer.parseInt(n.getName().replaceAll(String.format("%s\\((\\d*)\\).*", filename), "$1"));
if (actual>maxval)
maxval = actual;
}
String newName = ext.isEmpty()? String.format("%s(%d)", filename,maxval+1) : String.format("%s(%d).%s", filename,maxval+1, ext) ;
return Text.escapeIllegalJcrChars(newName);
} catch (Exception e) {
throw new BackendGenericError(e);
}
}
public static Node createFolderInternally(Session ses, Node destinationNode, String name, String description, boolean hidden, String login, AccountingHandler accountingHandler) throws BackendGenericError {
logger.debug("creating folder {} in {}", name, destinationNode);
String uniqueName = Utils.checkExistanceAndGetUniqueName(ses, destinationNode, name);
FolderItem item = new FolderItem();
Calendar now = Calendar.getInstance();
item.setName(uniqueName);
item.setTitle(uniqueName);
item.setDescription(description);
//item.setCreationTime(now);
boolean hiddenDestNode= false;
try {
hiddenDestNode = destinationNode.getProperty(NodeProperty.HIDDEN.toString()).getBoolean();
}catch (Throwable e) {}
item.setHidden(hidden || hiddenDestNode);
item.setLastAction(ItemAction.CREATED);
item.setLastModificationTime(now);
item.setLastModifiedBy(login);
item.setOwner(login);
item.setPublicItem(false);
Node newNode = new Item2NodeConverter().getNode(destinationNode, item);
if (accountingHandler!=null) {
accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, destinationNode, false);
accountingHandler.createEntryCreate(item.getTitle(), ses, newNode, false);
}
return newNode;
}
public static Node createURLInternally(Session ses, Node destinationNode, String name, URL value, String description, String login, AccountingHandler accountingHandler) throws BackendGenericError {
String uniqueName = Utils.checkExistanceAndGetUniqueName(ses, destinationNode, name);
ExternalLink item = new ExternalLink();
Calendar now = Calendar.getInstance();
item.setName(uniqueName);
item.setTitle(uniqueName);
item.setDescription(description);
//item.setCreationTime(now);
item.setLastAction(ItemAction.CREATED);
item.setLastModificationTime(now);
item.setLastModifiedBy(login);
item.setOwner(login);
item.setPublicItem(false);
item.setValue(value);
try {
item.setHidden(destinationNode.getProperty(NodeProperty.HIDDEN.toString()).getBoolean());
} catch (Throwable e) {
item.setHidden(false);
}
Node newNode = new Item2NodeConverter().getNode(destinationNode, item);
if (accountingHandler!=null) {
accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, destinationNode, false);
accountingHandler.createEntryCreate(item.getTitle(), ses, newNode, false);
}
return newNode;
}
public static Node createGcubeItemInternally(Session ses, Node destinationNode, String name, String description, String login, GCubeItem gcubeItem, AccountingHandler accountingHandler) throws BackendGenericError {
Calendar now = Calendar.getInstance();
gcubeItem.setName(name);
gcubeItem.setTitle(name);
gcubeItem.setDescription(description);
//item.setCreationTime(now);
gcubeItem.setHidden(false);
gcubeItem.setLastAction(ItemAction.CREATED);
gcubeItem.setLastModificationTime(now);
gcubeItem.setLastModifiedBy(login);
gcubeItem.setOwner(login);
//to inherit hidden property
//item.setHidden(destinationItem.isHidden());
Node newNode = new Item2NodeConverter().getNode(destinationNode, gcubeItem);
//TODO: accounting for GCUBEITEM
//accountingHandler.createFolderAddObj(name, item.getClass().getSimpleName(), null, ses, newNode, false);
return newNode;
}
public static void setPropertyOnChangeNode(Node node, String login, ItemAction action) throws RepositoryException {
node.setProperty(NodeProperty.LAST_MODIFIED.toString(), Calendar.getInstance());
node.setProperty(NodeProperty.LAST_MODIFIED_BY.toString(), login);
node.setProperty(NodeProperty.LAST_ACTION.toString(), action.name());
}
public static synchronized VRE getVreFolderItem(Session ses, Node2ItemConverter node2Item, VREManager vreManager, List<String> excludes ) throws RepositoryException, BackendGenericError{
org.gcube.common.storagehub.model.Path vrePath = Paths.append(Utils.getWorkspacePath(), Constants.VRE_FOLDER_PARENT_NAME);
ScopeBean bean = new ScopeBean(ScopeProvider.instance.get());
if (!bean.is(Type.VRE)) throw new BackendGenericError("the current scope is not a VRE");
String entireScopeName= bean.toString().replaceAll("^/(.*)/?$", "$1").replaceAll("/", "-");
VRE vre = vreManager.getVRE(entireScopeName);
if (vre!=null) return vre;
else {
String query = String.format("SELECT * FROM [nthl:workspaceItem] As node WHERE node.[jcr:title] like '%s' AND ISDESCENDANTNODE('%s')",entireScopeName, vrePath.toPath());
Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(query, Constants.QUERY_LANGUAGE);
NodeIterator it = jcrQuery.execute().getNodes();
if (!it.hasNext()) throw new BackendGenericError("vre folder not found for context "+entireScopeName);
Node folder = it.nextNode();
Item vreFolder = node2Item.getItem(folder, excludes);
return vreManager.putVRE(vreFolder);
}
}
public static synchronized VRE getVreFolderItemByGroupNameAndUser(Session ses, String goupName, String userId, Node2ItemConverter node2Item, VREManager vreManager, List<String> excludes ) throws RepositoryException, BackendGenericError{
org.gcube.common.storagehub.model.Path vrePath = Paths.append(Utils.getWorkspacePath(userId), Constants.VRE_FOLDER_PARENT_NAME);
VRE vre = vreManager.getVRE(goupName);
if (vre!=null) return vre;
else {
String query = String.format("SELECT * FROM [nthl:workspaceItem] As node WHERE node.[jcr:title] like '%s' AND ISDESCENDANTNODE('%s')",goupName, vrePath.toPath());
Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(query, Constants.QUERY_LANGUAGE);
NodeIterator it = jcrQuery.execute().getNodes();
if (!it.hasNext()) throw new BackendGenericError("vre folder not found for context "+goupName);
Node folder = it.nextNode();
Item vreFolder = node2Item.getItem(folder, excludes);
return vreManager.putVRE(vreFolder);
}
}
}