From 68eb65e1681d18d3f86617c45797da05c29f7881 Mon Sep 17 00:00:00 2001 From: lucio Date: Mon, 4 Oct 2021 17:11:43 +0200 Subject: [PATCH 1/8] removing query for retreiving VRE root Folder --- pom.xml | 2 +- .../storagehub/handlers/vres/VREManager.java | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index bd0fb1f..5426716 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ 4.0.0 org.gcube.data.access storagehub - 1.3.1 + 1.3.2-SNAPSHOT storagehub diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java index 432db4e..685f724 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java @@ -8,6 +8,7 @@ import java.util.concurrent.Executors; import javax.inject.Inject; import javax.inject.Singleton; +import javax.jcr.ItemNotFoundException; import javax.jcr.Node; import javax.jcr.NodeIterator; import javax.jcr.RepositoryException; @@ -101,14 +102,24 @@ public class VREManager { VRE vre = this.getVRE(groupName); if (vre!=null) return vre; else { - String query = String.format("SELECT * FROM [nthl:workspaceItem] As node WHERE node.[jcr:title] like '%s' AND ISDESCENDANTNODE('%s')",groupName, vrePath.toPath()); - Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(query, Constants.QUERY_LANGUAGE); - NodeIterator it = jcrQuery.execute().getNodes(); + + Node shareNode = ses.getNode(Constants.SHARED_FOLDER_PATH); + + NodeIterator it = shareNode.getNodes(); - if (!it.hasNext()) throw new BackendGenericError("vre folder not found for context "+groupName); + Node vreFolderNode = null; + + while (it.hasNext()) { + Node nextNode = it.nextNode(); + if (nextNode.hasProperty("jcr:title") && nextNode.getProperty("jcr:title").getString().equals(groupName)) { + vreFolderNode = nextNode; + break; + } + } + + if (vreFolderNode==null) throw new ItemNotFoundException("vre folder not found for context "+groupName); - Node folder = it.nextNode(); - Item vreFolder = node2Item.getItem(folder, excludes); + Item vreFolder = node2Item.getItem(vreFolderNode, excludes); return this.putVRE(vreFolder); } } From f30029c052c9c9843b99d555e25e1a69233c7a8e Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Wed, 6 Oct 2021 10:37:14 +0200 Subject: [PATCH 2/8] - using groupHandler method to retrieve VRE - added logs to ScriptManager error --- .../storagehub/handlers/GroupHandler.java | 27 ++++++++++--------- .../storagehub/handlers/vres/VREManager.java | 23 ++++------------ .../storagehub/services/GroupManager.java | 12 ++++----- .../services/admin/ScriptManager.java | 2 +- 4 files changed, 26 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java b/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java index 7b15df5..600273a 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/GroupHandler.java @@ -2,6 +2,7 @@ package org.gcube.data.access.storagehub.handlers; import javax.inject.Inject; import javax.inject.Singleton; +import javax.jcr.ItemNotFoundException; import javax.jcr.Node; import javax.jcr.NodeIterator; import javax.jcr.PathNotFoundException; @@ -24,10 +25,10 @@ import org.slf4j.LoggerFactory; public class GroupHandler { private static final Logger log = LoggerFactory.getLogger(GroupManager.class); - + @Inject PathUtil pathUtil; - + public boolean removeUserFromGroup(String groupId, String userId, JackrabbitSession session) throws StorageHubException, RepositoryException { org.apache.jackrabbit.api.security.user.UserManager usrManager = session.getUserManager(); @@ -39,7 +40,7 @@ public class GroupHandler { //delete folder on user String folderName = group.getPrincipal().getName(); - Node folder = getVreFolderNode(session, folderName); + Node folder = getFolderNodeRelatedToGroup(session, folderName); NodeIterator ni = folder.getSharedSet(); while (ni.hasNext()) { @@ -53,8 +54,8 @@ public class GroupHandler { return group.removeMember(user); } - - public Node getVreFolderNode(JackrabbitSession session, String name) throws InvalidItemException, RepositoryException { + + public Node getFolderNodeRelatedToGroup(JackrabbitSession session, String name) throws ItemNotFoundException, RepositoryException { Node sharedRootNode = session.getNode(Constants.SHARED_FOLDER_PATH); Node vreFolder = null; @@ -64,16 +65,18 @@ public class GroupHandler { log.debug("is an old HL VRE"); } - NodeIterator nodes = sharedRootNode.getNodes(); - while (nodes.hasNext()) { - Node node = nodes.nextNode(); - if (node.hasProperty(NodeProperty.TITLE.toString()) && node.getProperty(NodeProperty.TITLE.toString()).getString().equals(name)) { - vreFolder= node; - break; + if (vreFolder==null) { + NodeIterator nodes = sharedRootNode.getNodes(); + while (nodes.hasNext()) { + Node node = nodes.nextNode(); + if (node.hasProperty(NodeProperty.TITLE.toString()) && node.getProperty(NodeProperty.TITLE.toString()).getString().equals(name)) { + vreFolder= node; + break; + } } } - if (vreFolder==null) throw new InvalidItemException("vre folder not found"); + if (vreFolder==null) throw new ItemNotFoundException("vre folder not found for group "+name); return vreFolder; } diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java index 685f724..ebae0d0 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java @@ -12,9 +12,7 @@ import javax.jcr.ItemNotFoundException; import javax.jcr.Node; import javax.jcr.NodeIterator; import javax.jcr.RepositoryException; -import javax.jcr.Session; import javax.jcr.SimpleCredentials; -import javax.jcr.query.Query; import javax.servlet.ServletContext; import org.apache.jackrabbit.api.JackrabbitSession; @@ -30,6 +28,7 @@ import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.common.storagehub.model.items.Item; import org.gcube.data.access.storagehub.Constants; import org.gcube.data.access.storagehub.PathUtil; +import org.gcube.data.access.storagehub.handlers.GroupHandler; import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter; import org.gcube.data.access.storagehub.services.RepositoryInitializer; import org.slf4j.Logger; @@ -51,6 +50,9 @@ public class VREManager { @Inject PathUtil pathUtil; + @Inject + GroupHandler groupHandler; + ExecutorService executor = Executors.newFixedThreadPool(5); SimpleCredentials credentials; @@ -98,26 +100,11 @@ public class VREManager { throw new InvalidCallParameters("error getting VRE with user "+userId+" for group "+groupName); - org.gcube.common.storagehub.model.Path vrePath = pathUtil.getVREsPath(userId, ses); VRE vre = this.getVRE(groupName); if (vre!=null) return vre; else { - Node shareNode = ses.getNode(Constants.SHARED_FOLDER_PATH); - - NodeIterator it = shareNode.getNodes(); - - Node vreFolderNode = null; - - while (it.hasNext()) { - Node nextNode = it.nextNode(); - if (nextNode.hasProperty("jcr:title") && nextNode.getProperty("jcr:title").getString().equals(groupName)) { - vreFolderNode = nextNode; - break; - } - } - - if (vreFolderNode==null) throw new ItemNotFoundException("vre folder not found for context "+groupName); + Node vreFolderNode = groupHandler.getFolderNodeRelatedToGroup(ses, groupName); Item vreFolder = node2Item.getItem(vreFolderNode, excludes); return this.putVRE(vreFolder); diff --git a/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java b/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java index d27063e..8a716c7 100644 --- a/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/services/GroupManager.java @@ -9,7 +9,6 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; -import javax.enterprise.context.RequestScoped; import javax.inject.Inject; import javax.jcr.Node; import javax.jcr.PathNotFoundException; @@ -40,7 +39,6 @@ import org.apache.jackrabbit.api.security.user.QueryBuilder; import org.apache.jackrabbit.api.security.user.User; import org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils; import org.gcube.common.authorization.control.annotations.AuthorizationControl; -import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.gxrest.response.outbound.GXOutboundErrorResponse; import org.gcube.common.scope.api.ScopeProvider; @@ -200,11 +198,11 @@ public class GroupManager { org.apache.jackrabbit.api.security.user.UserManager usrManager = session.getUserManager(); Authorizable authorizable = usrManager.getAuthorizable(group); - if (authorizable.isGroup()) + if (authorizable!=null && authorizable.isGroup()) authorizable.remove(); try { - Node node = groupHandler.getVreFolderNode(session, group); + Node node = groupHandler.getFolderNodeRelatedToGroup(session, group); List workspaceItems = Utils.getItemList(node, Excludes.GET_ONLY_CONTENT, null, true, null); trashHandler.removeOnlyNodesContent(session, workspaceItems); node.removeSharedSet(); @@ -242,7 +240,7 @@ public class GroupManager { session = (JackrabbitSession) repository.getRepository().login(CredentialHandler.getAdminCredentials(context)); - Node vreFolder = groupHandler.getVreFolderNode(session, groupId); + Node vreFolder = groupHandler.getFolderNodeRelatedToGroup(session, groupId); String currentUser = AuthorizationProvider.instance.get().getClient().getId(); if (!isInfraManager() && !(isVREManager() && isValidGroupForContext(groupId) )) @@ -296,7 +294,7 @@ public class GroupManager { Objects.nonNull(userId); session = (JackrabbitSession) repository.getRepository().login(CredentialHandler.getAdminCredentials(context)); - Node vreFolder = groupHandler.getVreFolderNode(session, groupId); + Node vreFolder = groupHandler.getFolderNodeRelatedToGroup(session, groupId); String currentUser = AuthorizationProvider.instance.get().getClient().getId(); if (!isInfraManager() && !(isVREManager() && isValidGroupForContext(groupId) )) @@ -422,7 +420,7 @@ public class GroupManager { boolean success = group.addMember(user); session.save(); String folderName = group.getPrincipal().getName(); - Node folder = groupHandler.getVreFolderNode(session, folderName); + Node folder = groupHandler.getFolderNodeRelatedToGroup(session, folderName); String userPath = Paths.append(pathUtil.getVREsPath(user.getPrincipal().getName(), session), folderName).toPath(); log.debug("creating folder in user path {} from {}", userPath, folder.getPath() ); diff --git a/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java b/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java index a164b93..3bebb18 100644 --- a/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java @@ -168,7 +168,7 @@ public class ScriptManager { .stream(stream).on(parentId).with(ses).author(login); itemHandler.create(builder.build()); } catch (Throwable e) { - log.error("error saving script result {} in the Workspace",name); + log.error("error saving script result {} in the Workspace",name, e); } } finally { From 51cbd0776aec6cc518ce05695bbb929247cc2c10 Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Wed, 6 Oct 2021 11:44:23 +0200 Subject: [PATCH 3/8] solved bug https://support.d4science.org/issues/22147 --- .../data/access/storagehub/handlers/items/ItemHandler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java b/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java index 07474fd..87388d0 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java @@ -267,7 +267,10 @@ public class ItemHandler { authChecker.checkWriteAuthorizationControl(ses, login, newNode.getIdentifier(), false); AbstractFileItem item = fillItemWithContent(stream, storageBackend, name, description, relativePath,login); - item.setHidden(destinationNode.getProperty(NodeProperty.HIDDEN.toString()).getBoolean()); + + if (destinationNode.hasProperty(NodeProperty.HIDDEN.toString())) + item.setHidden(destinationNode.getProperty(NodeProperty.HIDDEN.toString()).getBoolean()); + if (withLock) { try { ses.getWorkspace().getLockManager().lock(newNode.getPath(), true, true, 0,login); From 7ebd1ae6295c0d9cb0c16997e555fe68c415f61e Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Wed, 6 Oct 2021 11:56:57 +0200 Subject: [PATCH 4/8] fix updated --- .../data/access/storagehub/handlers/items/ItemHandler.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java b/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java index 87388d0..6c2a2a0 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java @@ -267,10 +267,7 @@ public class ItemHandler { authChecker.checkWriteAuthorizationControl(ses, login, newNode.getIdentifier(), false); AbstractFileItem item = fillItemWithContent(stream, storageBackend, name, description, relativePath,login); - - if (destinationNode.hasProperty(NodeProperty.HIDDEN.toString())) - item.setHidden(destinationNode.getProperty(NodeProperty.HIDDEN.toString()).getBoolean()); - + if (withLock) { try { ses.getWorkspace().getLockManager().lock(newNode.getPath(), true, true, 0,login); From 586d9df93929657cf4c4010072f074e64202346a Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Wed, 6 Oct 2021 19:18:34 +0200 Subject: [PATCH 5/8] removed a part of query to retrieve recents --- .../access/storagehub/handlers/vres/VRE.java | 5 +- .../storagehub/handlers/vres/VREManager.java | 4 +- .../handlers/vres/VREQueryRetriever.java | 189 +++++++++++------- .../services/admin/ScriptUtilImpl.java | 3 +- 4 files changed, 119 insertions(+), 82 deletions(-) diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java index 802d33b..c3abb35 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java @@ -9,6 +9,7 @@ import javax.jcr.SimpleCredentials; import org.gcube.common.storagehub.model.exceptions.BackendGenericError; import org.gcube.common.storagehub.model.items.Item; +import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,11 +22,11 @@ public class VRE { private VREQueryRetriever vreQueryRetriever; private ExecutorService executor; - protected VRE(Item item, Repository repository, SimpleCredentials credentials, ExecutorService executor) { + protected VRE(Item item, Repository repository, SimpleCredentials credentials, Node2ItemConverter node2Item, ExecutorService executor) { super(); this.vreFolder = item; this.executor = executor; - vreQueryRetriever = new VREQueryRetriever(repository, credentials, vreFolder); + vreQueryRetriever = new VREQueryRetriever(repository, credentials, node2Item, vreFolder); result = executor.submit(vreQueryRetriever); } diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java index ebae0d0..cc1cab2 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java @@ -8,9 +8,7 @@ import java.util.concurrent.Executors; import javax.inject.Inject; import javax.inject.Singleton; -import javax.jcr.ItemNotFoundException; import javax.jcr.Node; -import javax.jcr.NodeIterator; import javax.jcr.RepositoryException; import javax.jcr.SimpleCredentials; import javax.servlet.ServletContext; @@ -76,7 +74,7 @@ public class VREManager { logger.trace("inserting VRE {}",vreFolder.getTitle()); if (vreMap.containsKey(vreFolder.getTitle())) throw new RuntimeException("something went wrong (vre already present in the map)"); else { - VRE toReturn = new VRE(vreFolder, repository.getRepository(), credentials, executor); + VRE toReturn = new VRE(vreFolder, repository.getRepository(), credentials, node2Item, executor); vreMap.put(vreFolder.getTitle(), toReturn); return toReturn; } diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java index 1a77bf8..2d37b2c 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java @@ -1,29 +1,33 @@ package org.gcube.data.access.storagehub.handlers.vres; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Iterator; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; -import java.util.Locale; +import java.util.Map; import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import javax.inject.Inject; import javax.jcr.Credentials; import javax.jcr.Node; +import javax.jcr.NodeIterator; import javax.jcr.Property; import javax.jcr.Repository; import javax.jcr.RepositoryException; import javax.jcr.Session; import javax.jcr.observation.Event; import javax.jcr.observation.EventJournal; -import javax.jcr.query.Query; -import org.apache.jackrabbit.util.ISO9075; import org.gcube.common.storagehub.model.Excludes; import org.gcube.common.storagehub.model.NodeConstants; +import org.gcube.common.storagehub.model.exceptions.BackendGenericError; +import org.gcube.common.storagehub.model.items.AbstractFileItem; +import org.gcube.common.storagehub.model.items.FolderItem; import org.gcube.common.storagehub.model.items.Item; -import org.gcube.data.access.storagehub.NodeChildrenFilterIterator; import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,64 +41,32 @@ public class VREQueryRetriever implements Callable> { private Repository repository; private Credentials credentials; private Item vreFolder; - List cachedList = new ArrayList<>(CACHE_DIMENSION); + Map cachedList = new HashMap<>(CACHE_DIMENSION); + long higherTimestamp = Long.MAX_VALUE; + long lastTimestamp =0; - private Node2ItemConverter node2Item = new Node2ItemConverter(); + Node2ItemConverter node2Item; - public VREQueryRetriever(Repository repository, Credentials credentials, Item vreFolder) { + public VREQueryRetriever(Repository repository, Credentials credentials, Node2ItemConverter node2Item, Item vreFolder) { super(); this.repository = repository; this.credentials = credentials; this.vreFolder = vreFolder; + this.node2Item = node2Item; } public List call() { - logger.trace("executing recents task"); + logger.debug("executing recents task for {}",vreFolder.getTitle()); Session ses = null; - if (lastTimestamp==0) { - try { - long start = System.currentTimeMillis(); - ses = repository.login(credentials); - - Calendar now = Calendar.getInstance(); - now.add(Calendar.YEAR, -1); - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS", Locale.ENGLISH); - String formattedDate = formatter.format(now.getTime()); - - - String xpath = String.format("/jcr:root%s//element(*,nthl:workspaceLeafItem)[@jcr:lastModified>xs:dateTime('%s')] order by @jcr:lastModified descending",ISO9075.encodePath(vreFolder.getPath()),formattedDate); - - //String query = String.format("SELECT * FROM [nthl:workspaceLeafItem] AS node WHERE ISDESCENDANTNODE('%s') ORDER BY node.[jcr:lastModified] DESC ",vreFolder.getPath()); - logger.debug("query for recents is {}",xpath); - - Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(xpath, Query.XPATH); - - jcrQuery.setLimit(CACHE_DIMENSION); - lastTimestamp = System.currentTimeMillis(); - NodeChildrenFilterIterator it = new NodeChildrenFilterIterator(jcrQuery.execute().getNodes()); - logger.debug("query for recents took {}",System.currentTimeMillis()-start); - while (it.hasNext()) { - Node node = it.next(); - - Item item = node2Item.getItem(node, Excludes.EXCLUDE_ACCOUNTING); - logger.debug("RECENTS - adding item {} with timestamp {}",item.getTitle(), item.getLastModificationTime().getTimeInMillis()); - cachedList.add(item); - - } - logger.debug("creating objects took {}",System.currentTimeMillis()-start); - if (cachedList.size()<=10) return cachedList; - else return cachedList.subList(0, 10); - } catch (Exception e) { - logger.error("error querying vre {}",vreFolder.getTitle(),e); - throw new RuntimeException(e); - }finally{ - if (ses!=null) - ses.logout(); - logger.trace("recents task finished"); - } - } else { + try { + ses = repository.login(credentials); + //if (cachedList.isEmpty()) { + + init(ses); + /*} else { + logger.debug("redoing recents for {}",vreFolder.getTitle()); try { long timestampToUse = lastTimestamp; @@ -107,7 +79,7 @@ public class VREQueryRetriever implements Callable> { EventJournal journalChanged = ses.getWorkspace().getObservationManager().getEventJournal(Event.PROPERTY_CHANGED^Event.NODE_REMOVED^Event.NODE_MOVED^Event.NODE_ADDED, vreFolder.getPath(), true, null, types); journalChanged.skipTo(timestampToUse); - logger.trace("getting the journal took {}",System.currentTimeMillis()-start); + logger.debug("getting the journal took {}",System.currentTimeMillis()-start); int events = 0; @@ -122,6 +94,8 @@ public class VREQueryRetriever implements Callable> { if (nodeAdded.isNodeType("nthl:workspaceLeafItem")) { logger.trace("node added event received with name {}", nodeAdded.getName()); Item item = node2Item.getItem(nodeAdded, Arrays.asList(NodeConstants.ACCOUNTING_NAME)); + if (cachedList.get(event.getIdentifier())!=null) + cachedList.remove(event.getIdentifier()); insertItemInTheRightPlace(item); } } @@ -133,7 +107,7 @@ public class VREQueryRetriever implements Callable> { if (property.getName().equalsIgnoreCase("jcr:lastModified")) { logger.trace("event property changed on {} with value {} and parent {}",property.getName(), property.getValue().getString(), property.getParent().getPath()); String identifier = property.getParent().getIdentifier(); - cachedList.removeIf(i -> i.getId().equals(identifier)); + cachedList.remove(identifier); Item item = node2Item.getItem(property.getParent(), Excludes.EXCLUDE_ACCOUNTING); insertItemInTheRightPlace(item); } @@ -141,14 +115,10 @@ public class VREQueryRetriever implements Callable> { break; case Event.NODE_REMOVED: logger.trace("node removed event received with type {}", event.getIdentifier()); - cachedList.removeIf(i -> { - try { - return i.getId().equals(event.getIdentifier()) && i.getLastModificationTime().getTime().getTime()> { if (nodeMoved.isNodeType("nthl:workspaceLeafItem")) { logger.trace("event node moved on {} with path {}",nodeMoved.getName(), nodeMoved.getPath()); String identifier = nodeMoved.getIdentifier(); - cachedList.removeIf(i -> i.getId().equals(identifier) && !i.getPath().startsWith(vreFolder.getPath())); + String nodePath = ses.getNode(identifier).getPath(); + if (cachedList.get(event.getIdentifier())!=null && + !nodePath.startsWith(vreFolder.getPath())) + cachedList.remove(event.getIdentifier()); + } break; default: @@ -165,11 +139,9 @@ public class VREQueryRetriever implements Callable> { } - if (cachedList.size()>CACHE_DIMENSION) - cachedList.subList(51, cachedList.size()).clear(); + logger.trace("retrieving event took {} with {} events",System.currentTimeMillis()-start, events); - if (cachedList.size()<=10) return cachedList; - else return cachedList.subList(0, 10); + } catch (Exception e) { logger.error("error getting events for vre {}",vreFolder.getTitle(),e); throw new RuntimeException(e); @@ -177,20 +149,87 @@ public class VREQueryRetriever implements Callable> { if (ses!=null) ses.logout(); } + } */ + + return correctValues(ses); + } catch (Exception e) { + logger.error("error preparing recents for folder {}", vreFolder.getTitle(),e); + return Collections.emptyList(); + }finally{ + if (ses!=null) + ses.logout(); + logger.debug("recents task finished"); + } + } + + + + private List correctValues(Session ses){ + logger.debug("preparing returning values for {}",vreFolder.getTitle()); + long start = System.currentTimeMillis(); + List> list = new LinkedList<>(cachedList.entrySet()); + list.sort((c1, c2) -> c1.getValue().compareTo(c2.getValue())*-1); + if (list.size()>CACHE_DIMENSION) + for (int index = CACHE_DIMENSION-1; index< list.size() ; index++) + cachedList.remove(list.get(index).getKey()); + + List cachedIds = list.stream().map(m -> m.getKey()).collect(Collectors.toList()); + if (cachedIds.size()>10) + cachedIds = cachedIds.subList(0, 10); + + List result = new ArrayList<>(10); + + for (String id: cachedIds) { + try { + Item item = node2Item.getItem(id, ses, Excludes.EXCLUDE_ACCOUNTING); + if (item!=null) + result.add(item); + else logger.warn("item with id {} is null",id); + } catch (BackendGenericError | RepositoryException e) { } + } + + logger.debug("returning values prepared in {} for {}",System.currentTimeMillis()-start,vreFolder.getTitle()); + + return result; + } + + private void insertItemInTheRightPlace(Item item) { + long lastModifiedTime = item.getLastModificationTime().getTime().getTime(); + + if (!(lastModifiedTime>higherTimestamp && cachedList.size()>CACHE_DIMENSION)) { + cachedList.put(item.getId(), lastModifiedTime); + if (lastModifiedTime>higherTimestamp) higherTimestamp = lastModifiedTime; } } - private void insertItemInTheRightPlace(Item item) { - Iterator it = cachedList.iterator(); - int index =0; - while (it.hasNext()) { - Item inListItem = it.next(); - if (item.getLastModificationTime().getTime().getTime()>=inListItem.getLastModificationTime().getTime().getTime()) break; - index++; + private void init(Session ses){ + try { + long start = System.currentTimeMillis(); + Calendar now = Calendar.getInstance(); + now.add(Calendar.YEAR, -1); + lastTimestamp = System.currentTimeMillis(); + Node vreFolderNode = ses.getNodeByIdentifier(vreFolder.getId()); + logger.debug("starting visiting children for {}",vreFolder.getTitle()); + visitChildren(vreFolderNode); + logger.debug("initializing recents for {} took {}",vreFolder.getTitle(),System.currentTimeMillis()-start); + } catch (Exception e) { + logger.error("error querying vre {}",vreFolder.getTitle(),e); + throw new RuntimeException(e); + } + } + + private void visitChildren(Node node) throws Exception{ + NodeIterator nodeIt = node.getNodes(); + while(nodeIt.hasNext()) { + Node child = nodeIt.nextNode(); + Item item = node2Item.getItem(child, Excludes.ALL); + if (item==null || item.isHidden()) continue; + if (item instanceof FolderItem) + visitChildren(child); + else if(item instanceof AbstractFileItem) + insertItemInTheRightPlace(item); } - if (index getChildren(Predicate checker, Node parent, List excludes, boolean showHidden, Class nodeTypeToInclude) throws RepositoryException, BackendGenericError { - return Utils.getItemList(checker, parent, excludes, new Range(0, 100), showHidden, nodeTypeToInclude); + return Utils.getItemList(checker, parent, excludes, null, showHidden, nodeTypeToInclude); } @Override From 27c6f2d3e1027ae525d919970c96760b26889073 Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Wed, 6 Oct 2021 19:41:18 +0200 Subject: [PATCH 6/8] scriptManager accept different names --- .../data/access/storagehub/services/admin/ScriptManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java b/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java index 3bebb18..1d1fd00 100644 --- a/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/services/admin/ScriptManager.java @@ -74,7 +74,7 @@ public class ScriptManager { @FormDataParam("file") FormDataContentDisposition fileDetail) { try { ScriptClassLoader scriptClassLoader = new ScriptClassLoader(Thread.currentThread().getContextClassLoader()); - Class scriptClass = uploadClass(stream, scriptClassLoader, name); + Class scriptClass = uploadClass(stream, scriptClassLoader, fileDetail.getFileName().replace(".class", "")); return run(scriptClass, name, destinationFolderId, asynch!=null? asynch : false); }catch(Throwable e) { log.error("error executing script {}", name,e); From 47bf9b57d24f3f38e3e1c80a9e64fc63beb68bb0 Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Thu, 7 Oct 2021 10:38:35 +0200 Subject: [PATCH 7/8] pom and changelog update --- CHANGELOG.md | 8 ++++++++ pom.xml | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d8b74a..07f9ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v1.4.0] - [2021-10-07] + +- slow query removed from VRE retrieving and recents + +## [v1.3.1] - [2021-09-08] + +- solved bug on attachment rights + ## [v1.3.1] - [2021-09-08] - solved bug on attachment rights diff --git a/pom.xml b/pom.xml index 5426716..68a2162 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ 4.0.0 org.gcube.data.access storagehub - 1.3.2-SNAPSHOT + 1.4.0-SNAPSHOT storagehub @@ -300,13 +300,13 @@ org.gcube.contentmanagement storage-manager-core - [2.0.0,3.0.0-SNAPSHOT) + [3.0.0-SNAPSHOT,4.0.0-SNAPSHOT) org.gcube.contentmanagement storage-manager-wrapper - [2.0.0,3.0.0-SNAPSHOT) + [3.0.0-SNAPSHOT,4.0.0-SNAPSHOT) From 20390e3147e114e905a9145f6e99cbef8d46c9c9 Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Tue, 12 Oct 2021 13:12:02 +0200 Subject: [PATCH 8/8] incident https://support.d4science.org/issues/22184 solved --- CHANGELOG.md | 1 + .../handlers/items/Node2ItemConverter.java | 3 + .../handlers/vres/VREQueryRetriever.java | 238 ++++++++++-------- .../storagehub/services/MessageManager.java | 6 + 4 files changed, 140 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f9ff1..a8974f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [v1.4.0] - [2021-10-07] - slow query removed from VRE retrieving and recents +- incident #22184 solved ## [v1.3.1] - [2021-09-08] diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java b/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java index 65491df..141743d 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java @@ -85,6 +85,9 @@ public class Node2ItemConverter { } public Message getMessageItem(Node node) throws RepositoryException{ + if (!(node.getPrimaryNodeType().getName().equals("nthl:itemSentRequest") + || node.getPrimaryNodeType().getName().equals("nthl:itemSentRequestSH"))) + return null; Message msg = new Message(); try { Node attachmentNode = node.getNode(Constants.ATTACHMENTNODE_NAME); diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java index 2d37b2c..a95910a 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java @@ -1,7 +1,6 @@ package org.gcube.data.access.storagehub.handlers.vres; import java.util.ArrayList; -import java.util.Arrays; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; @@ -11,7 +10,6 @@ import java.util.Map; import java.util.concurrent.Callable; import java.util.stream.Collectors; -import javax.inject.Inject; import javax.jcr.Credentials; import javax.jcr.Node; import javax.jcr.NodeIterator; @@ -23,7 +21,6 @@ import javax.jcr.observation.Event; import javax.jcr.observation.EventJournal; import org.gcube.common.storagehub.model.Excludes; -import org.gcube.common.storagehub.model.NodeConstants; import org.gcube.common.storagehub.model.exceptions.BackendGenericError; import org.gcube.common.storagehub.model.items.AbstractFileItem; import org.gcube.common.storagehub.model.items.FolderItem; @@ -41,14 +38,17 @@ public class VREQueryRetriever implements Callable> { private Repository repository; private Credentials credentials; private Item vreFolder; - Map cachedList = new HashMap<>(CACHE_DIMENSION); + Map cachedMap = new HashMap<>(CACHE_DIMENSION); long higherTimestamp = Long.MAX_VALUE; long lastTimestamp =0; + long doTime= 0; + boolean underRedo = false; Node2ItemConverter node2Item; + public VREQueryRetriever(Repository repository, Credentials credentials, Node2ItemConverter node2Item, Item vreFolder) { super(); this.repository = repository; @@ -58,101 +58,22 @@ public class VREQueryRetriever implements Callable> { } public List call() { - logger.debug("executing recents task for {}",vreFolder.getTitle()); Session ses = null; try { ses = repository.login(credentials); - //if (cachedList.isEmpty()) { - - init(ses); - /*} else { - logger.debug("redoing recents for {}",vreFolder.getTitle()); - try { - - long timestampToUse = lastTimestamp; - lastTimestamp = System.currentTimeMillis(); - - long start = System.currentTimeMillis(); - ses = repository.login(credentials); - final String[] types = { "nthl:workspaceLeafItem", "nthl:workspaceItem"}; - - EventJournal journalChanged = ses.getWorkspace().getObservationManager().getEventJournal(Event.PROPERTY_CHANGED^Event.NODE_REMOVED^Event.NODE_MOVED^Event.NODE_ADDED, vreFolder.getPath(), true, null, types); - journalChanged.skipTo(timestampToUse); - - logger.debug("getting the journal took {}",System.currentTimeMillis()-start); - - int events = 0; - - while (journalChanged.hasNext()) { - events++; - Event event = journalChanged.nextEvent(); - switch(event.getType()) { - - case Event.NODE_ADDED: - if (ses.nodeExists(event.getPath())) { - Node nodeAdded = ses.getNode(event.getPath()); - if (nodeAdded.isNodeType("nthl:workspaceLeafItem")) { - logger.trace("node added event received with name {}", nodeAdded.getName()); - Item item = node2Item.getItem(nodeAdded, Arrays.asList(NodeConstants.ACCOUNTING_NAME)); - if (cachedList.get(event.getIdentifier())!=null) - cachedList.remove(event.getIdentifier()); - insertItemInTheRightPlace(item); - } - } - break; - - case Event.PROPERTY_CHANGED: - if (ses.propertyExists(event.getPath())) { - Property property = ses.getProperty(event.getPath()); - if (property.getName().equalsIgnoreCase("jcr:lastModified")) { - logger.trace("event property changed on {} with value {} and parent {}",property.getName(), property.getValue().getString(), property.getParent().getPath()); - String identifier = property.getParent().getIdentifier(); - cachedList.remove(identifier); - Item item = node2Item.getItem(property.getParent(), Excludes.EXCLUDE_ACCOUNTING); - insertItemInTheRightPlace(item); - } - } - break; - case Event.NODE_REMOVED: - logger.trace("node removed event received with type {}", event.getIdentifier()); - - if (cachedList.get(event.getIdentifier())!=null && - cachedList.get(event.getIdentifier()) now.getTimeInMillis() || underRedo) { + logger.debug("executing recents task for {} (cahced result)",vreFolder.getTitle()); + return correctValues(ses, cachedMap); + }else { + logger.debug("executing recents task for {} (redoing it)",vreFolder.getTitle()); + List toReturn = redo(ses); + doTime = System.currentTimeMillis(); + return toReturn; } - } */ - - return correctValues(ses); - } catch (Exception e) { + }catch (Exception e) { logger.error("error preparing recents for folder {}", vreFolder.getTitle(),e); return Collections.emptyList(); }finally{ @@ -162,16 +83,32 @@ public class VREQueryRetriever implements Callable> { } } + public synchronized List redo(Session ses) { + underRedo = true; + try { + Map tempCachedMap = new HashMap<>(CACHE_DIMENSION); + if (cachedMap.isEmpty()) + init(ses, tempCachedMap); + else { + logger.debug("redoing recents for {}",vreFolder.getTitle()); + update(ses, tempCachedMap); + } + cachedMap = tempCachedMap; + return correctValues(ses, tempCachedMap); + }finally{ + underRedo = false; + } + } - private List correctValues(Session ses){ + private List correctValues(Session ses, Map tempCachedMap){ logger.debug("preparing returning values for {}",vreFolder.getTitle()); long start = System.currentTimeMillis(); - List> list = new LinkedList<>(cachedList.entrySet()); + List> list = new LinkedList<>(tempCachedMap.entrySet()); list.sort((c1, c2) -> c1.getValue().compareTo(c2.getValue())*-1); if (list.size()>CACHE_DIMENSION) for (int index = CACHE_DIMENSION-1; index< list.size() ; index++) - cachedList.remove(list.get(index).getKey()); + tempCachedMap.remove(list.get(index).getKey()); List cachedIds = list.stream().map(m -> m.getKey()).collect(Collectors.toList()); if (cachedIds.size()>10) @@ -193,25 +130,23 @@ public class VREQueryRetriever implements Callable> { return result; } - private void insertItemInTheRightPlace(Item item) { + private void insertItemInTheRightPlace(Item item, Map tempCachedMap) { long lastModifiedTime = item.getLastModificationTime().getTime().getTime(); - if (!(lastModifiedTime>higherTimestamp && cachedList.size()>CACHE_DIMENSION)) { - cachedList.put(item.getId(), lastModifiedTime); + if (!(lastModifiedTime>higherTimestamp && tempCachedMap.size()>CACHE_DIMENSION)) { + tempCachedMap.put(item.getId(), lastModifiedTime); if (lastModifiedTime>higherTimestamp) higherTimestamp = lastModifiedTime; } } - private void init(Session ses){ + private void init(Session ses,Map tempCachedMap){ try { long start = System.currentTimeMillis(); - Calendar now = Calendar.getInstance(); - now.add(Calendar.YEAR, -1); lastTimestamp = System.currentTimeMillis(); Node vreFolderNode = ses.getNodeByIdentifier(vreFolder.getId()); logger.debug("starting visiting children for {}",vreFolder.getTitle()); - visitChildren(vreFolderNode); + visitChildren(vreFolderNode, tempCachedMap); logger.debug("initializing recents for {} took {}",vreFolder.getTitle(),System.currentTimeMillis()-start); } catch (Exception e) { logger.error("error querying vre {}",vreFolder.getTitle(),e); @@ -219,16 +154,103 @@ public class VREQueryRetriever implements Callable> { } } - private void visitChildren(Node node) throws Exception{ + + private void update(Session ses, Map tempCachedMap) { + try { + long timestampToUse = lastTimestamp; + lastTimestamp = System.currentTimeMillis(); + + long start = System.currentTimeMillis(); + final String[] types = { "nthl:workspaceLeafItem", "nthl:workspaceItem"}; + + EventJournal journalChanged = ses.getWorkspace().getObservationManager().getEventJournal(Event.PROPERTY_CHANGED^Event.NODE_REMOVED^Event.NODE_MOVED^Event.NODE_ADDED, vreFolder.getPath(), true, null, types); + journalChanged.skipTo(timestampToUse); + + logger.debug("getting the journal took {}",System.currentTimeMillis()-start); + + int events = 0; + + //TODO: manage hidden nodes + while (journalChanged.hasNext()) { + events++; + Event event = journalChanged.nextEvent(); + try { + switch(event.getType()) { + + case Event.NODE_ADDED: + if (ses.nodeExists(event.getPath())) { + Node nodeAdded = ses.getNode(event.getPath()); + if (nodeAdded.isNodeType("nthl:workspaceLeafItem")) { + logger.debug("node added event received with name {}", nodeAdded.getName()); + Item item = node2Item.getItem(nodeAdded, Excludes.ALL); + if (tempCachedMap.get(event.getIdentifier())!=null) + tempCachedMap.remove(event.getIdentifier()); + insertItemInTheRightPlace(item,tempCachedMap); + } + } + break; + + case Event.PROPERTY_CHANGED: + if (ses.propertyExists(event.getPath())) { + Property property = ses.getProperty(event.getPath()); + if (property.getName().equalsIgnoreCase("jcr:lastModified")) { + logger.debug("event property changed on {} with value {} and parent {}",property.getName(), property.getValue().getString(), property.getParent().getPath()); + String identifier = property.getParent().getIdentifier(); + tempCachedMap.remove(identifier); + Item item = node2Item.getItem(property.getParent(), Excludes.ALL); + insertItemInTheRightPlace(item, tempCachedMap); + } + } + break; + case Event.NODE_REMOVED: + logger.trace("node removed event received with type {}", event.getIdentifier()); + + if (tempCachedMap.get(event.getIdentifier())!=null && + tempCachedMap.get(event.getIdentifier()) tempCachedMap) throws Exception{ NodeIterator nodeIt = node.getNodes(); while(nodeIt.hasNext()) { Node child = nodeIt.nextNode(); Item item = node2Item.getItem(child, Excludes.ALL); if (item==null || item.isHidden()) continue; if (item instanceof FolderItem) - visitChildren(child); + visitChildren(child,tempCachedMap); else if(item instanceof AbstractFileItem) - insertItemInTheRightPlace(item); + insertItemInTheRightPlace(item,tempCachedMap); } } diff --git a/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java b/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java index b101740..d407da5 100644 --- a/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java @@ -345,7 +345,13 @@ public class MessageManager extends Impersonable{ NodeIterator nodeIt = node.getNodes(); while(nodeIt.hasNext()) { Node child = nodeIt.nextNode(); + log.info("message type "+child.getPrimaryNodeType().getName()); Message message = node2Item.getMessageItem(child); + if (message == null) { + log.info("message discarded"); + continue; + } + if (reduceBody != null && reduceBody>0 && message.getBody().length()>reduceBody ) message.setBody(message.getBody().substring(0, reduceBody)); insertOrdered(messages, message);