From 20390e3147e114e905a9145f6e99cbef8d46c9c9 Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Tue, 12 Oct 2021 13:12:02 +0200 Subject: [PATCH] incident https://support.d4science.org/issues/22184 solved --- CHANGELOG.md | 1 + .../handlers/items/Node2ItemConverter.java | 3 + .../handlers/vres/VREQueryRetriever.java | 238 ++++++++++-------- .../storagehub/services/MessageManager.java | 6 + 4 files changed, 140 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f9ff1..a8974f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [v1.4.0] - [2021-10-07] - slow query removed from VRE retrieving and recents +- incident #22184 solved ## [v1.3.1] - [2021-09-08] diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java b/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java index 65491df..141743d 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/items/Node2ItemConverter.java @@ -85,6 +85,9 @@ public class Node2ItemConverter { } public Message getMessageItem(Node node) throws RepositoryException{ + if (!(node.getPrimaryNodeType().getName().equals("nthl:itemSentRequest") + || node.getPrimaryNodeType().getName().equals("nthl:itemSentRequestSH"))) + return null; Message msg = new Message(); try { Node attachmentNode = node.getNode(Constants.ATTACHMENTNODE_NAME); diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java index 2d37b2c..a95910a 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java @@ -1,7 +1,6 @@ package org.gcube.data.access.storagehub.handlers.vres; import java.util.ArrayList; -import java.util.Arrays; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; @@ -11,7 +10,6 @@ import java.util.Map; import java.util.concurrent.Callable; import java.util.stream.Collectors; -import javax.inject.Inject; import javax.jcr.Credentials; import javax.jcr.Node; import javax.jcr.NodeIterator; @@ -23,7 +21,6 @@ import javax.jcr.observation.Event; import javax.jcr.observation.EventJournal; import org.gcube.common.storagehub.model.Excludes; -import org.gcube.common.storagehub.model.NodeConstants; import org.gcube.common.storagehub.model.exceptions.BackendGenericError; import org.gcube.common.storagehub.model.items.AbstractFileItem; import org.gcube.common.storagehub.model.items.FolderItem; @@ -41,14 +38,17 @@ public class VREQueryRetriever implements Callable> { private Repository repository; private Credentials credentials; private Item vreFolder; - Map cachedList = new HashMap<>(CACHE_DIMENSION); + Map cachedMap = new HashMap<>(CACHE_DIMENSION); long higherTimestamp = Long.MAX_VALUE; long lastTimestamp =0; + long doTime= 0; + boolean underRedo = false; Node2ItemConverter node2Item; + public VREQueryRetriever(Repository repository, Credentials credentials, Node2ItemConverter node2Item, Item vreFolder) { super(); this.repository = repository; @@ -58,101 +58,22 @@ public class VREQueryRetriever implements Callable> { } public List call() { - logger.debug("executing recents task for {}",vreFolder.getTitle()); Session ses = null; try { ses = repository.login(credentials); - //if (cachedList.isEmpty()) { - - init(ses); - /*} else { - logger.debug("redoing recents for {}",vreFolder.getTitle()); - try { - - long timestampToUse = lastTimestamp; - lastTimestamp = System.currentTimeMillis(); - - long start = System.currentTimeMillis(); - ses = repository.login(credentials); - final String[] types = { "nthl:workspaceLeafItem", "nthl:workspaceItem"}; - - EventJournal journalChanged = ses.getWorkspace().getObservationManager().getEventJournal(Event.PROPERTY_CHANGED^Event.NODE_REMOVED^Event.NODE_MOVED^Event.NODE_ADDED, vreFolder.getPath(), true, null, types); - journalChanged.skipTo(timestampToUse); - - logger.debug("getting the journal took {}",System.currentTimeMillis()-start); - - int events = 0; - - while (journalChanged.hasNext()) { - events++; - Event event = journalChanged.nextEvent(); - switch(event.getType()) { - - case Event.NODE_ADDED: - if (ses.nodeExists(event.getPath())) { - Node nodeAdded = ses.getNode(event.getPath()); - if (nodeAdded.isNodeType("nthl:workspaceLeafItem")) { - logger.trace("node added event received with name {}", nodeAdded.getName()); - Item item = node2Item.getItem(nodeAdded, Arrays.asList(NodeConstants.ACCOUNTING_NAME)); - if (cachedList.get(event.getIdentifier())!=null) - cachedList.remove(event.getIdentifier()); - insertItemInTheRightPlace(item); - } - } - break; - - case Event.PROPERTY_CHANGED: - if (ses.propertyExists(event.getPath())) { - Property property = ses.getProperty(event.getPath()); - if (property.getName().equalsIgnoreCase("jcr:lastModified")) { - logger.trace("event property changed on {} with value {} and parent {}",property.getName(), property.getValue().getString(), property.getParent().getPath()); - String identifier = property.getParent().getIdentifier(); - cachedList.remove(identifier); - Item item = node2Item.getItem(property.getParent(), Excludes.EXCLUDE_ACCOUNTING); - insertItemInTheRightPlace(item); - } - } - break; - case Event.NODE_REMOVED: - logger.trace("node removed event received with type {}", event.getIdentifier()); - - if (cachedList.get(event.getIdentifier())!=null && - cachedList.get(event.getIdentifier()) now.getTimeInMillis() || underRedo) { + logger.debug("executing recents task for {} (cahced result)",vreFolder.getTitle()); + return correctValues(ses, cachedMap); + }else { + logger.debug("executing recents task for {} (redoing it)",vreFolder.getTitle()); + List toReturn = redo(ses); + doTime = System.currentTimeMillis(); + return toReturn; } - } */ - - return correctValues(ses); - } catch (Exception e) { + }catch (Exception e) { logger.error("error preparing recents for folder {}", vreFolder.getTitle(),e); return Collections.emptyList(); }finally{ @@ -162,16 +83,32 @@ public class VREQueryRetriever implements Callable> { } } + public synchronized List redo(Session ses) { + underRedo = true; + try { + Map tempCachedMap = new HashMap<>(CACHE_DIMENSION); + if (cachedMap.isEmpty()) + init(ses, tempCachedMap); + else { + logger.debug("redoing recents for {}",vreFolder.getTitle()); + update(ses, tempCachedMap); + } + cachedMap = tempCachedMap; + return correctValues(ses, tempCachedMap); + }finally{ + underRedo = false; + } + } - private List correctValues(Session ses){ + private List correctValues(Session ses, Map tempCachedMap){ logger.debug("preparing returning values for {}",vreFolder.getTitle()); long start = System.currentTimeMillis(); - List> list = new LinkedList<>(cachedList.entrySet()); + List> list = new LinkedList<>(tempCachedMap.entrySet()); list.sort((c1, c2) -> c1.getValue().compareTo(c2.getValue())*-1); if (list.size()>CACHE_DIMENSION) for (int index = CACHE_DIMENSION-1; index< list.size() ; index++) - cachedList.remove(list.get(index).getKey()); + tempCachedMap.remove(list.get(index).getKey()); List cachedIds = list.stream().map(m -> m.getKey()).collect(Collectors.toList()); if (cachedIds.size()>10) @@ -193,25 +130,23 @@ public class VREQueryRetriever implements Callable> { return result; } - private void insertItemInTheRightPlace(Item item) { + private void insertItemInTheRightPlace(Item item, Map tempCachedMap) { long lastModifiedTime = item.getLastModificationTime().getTime().getTime(); - if (!(lastModifiedTime>higherTimestamp && cachedList.size()>CACHE_DIMENSION)) { - cachedList.put(item.getId(), lastModifiedTime); + if (!(lastModifiedTime>higherTimestamp && tempCachedMap.size()>CACHE_DIMENSION)) { + tempCachedMap.put(item.getId(), lastModifiedTime); if (lastModifiedTime>higherTimestamp) higherTimestamp = lastModifiedTime; } } - private void init(Session ses){ + private void init(Session ses,Map tempCachedMap){ try { long start = System.currentTimeMillis(); - Calendar now = Calendar.getInstance(); - now.add(Calendar.YEAR, -1); lastTimestamp = System.currentTimeMillis(); Node vreFolderNode = ses.getNodeByIdentifier(vreFolder.getId()); logger.debug("starting visiting children for {}",vreFolder.getTitle()); - visitChildren(vreFolderNode); + visitChildren(vreFolderNode, tempCachedMap); logger.debug("initializing recents for {} took {}",vreFolder.getTitle(),System.currentTimeMillis()-start); } catch (Exception e) { logger.error("error querying vre {}",vreFolder.getTitle(),e); @@ -219,16 +154,103 @@ public class VREQueryRetriever implements Callable> { } } - private void visitChildren(Node node) throws Exception{ + + private void update(Session ses, Map tempCachedMap) { + try { + long timestampToUse = lastTimestamp; + lastTimestamp = System.currentTimeMillis(); + + long start = System.currentTimeMillis(); + final String[] types = { "nthl:workspaceLeafItem", "nthl:workspaceItem"}; + + EventJournal journalChanged = ses.getWorkspace().getObservationManager().getEventJournal(Event.PROPERTY_CHANGED^Event.NODE_REMOVED^Event.NODE_MOVED^Event.NODE_ADDED, vreFolder.getPath(), true, null, types); + journalChanged.skipTo(timestampToUse); + + logger.debug("getting the journal took {}",System.currentTimeMillis()-start); + + int events = 0; + + //TODO: manage hidden nodes + while (journalChanged.hasNext()) { + events++; + Event event = journalChanged.nextEvent(); + try { + switch(event.getType()) { + + case Event.NODE_ADDED: + if (ses.nodeExists(event.getPath())) { + Node nodeAdded = ses.getNode(event.getPath()); + if (nodeAdded.isNodeType("nthl:workspaceLeafItem")) { + logger.debug("node added event received with name {}", nodeAdded.getName()); + Item item = node2Item.getItem(nodeAdded, Excludes.ALL); + if (tempCachedMap.get(event.getIdentifier())!=null) + tempCachedMap.remove(event.getIdentifier()); + insertItemInTheRightPlace(item,tempCachedMap); + } + } + break; + + case Event.PROPERTY_CHANGED: + if (ses.propertyExists(event.getPath())) { + Property property = ses.getProperty(event.getPath()); + if (property.getName().equalsIgnoreCase("jcr:lastModified")) { + logger.debug("event property changed on {} with value {} and parent {}",property.getName(), property.getValue().getString(), property.getParent().getPath()); + String identifier = property.getParent().getIdentifier(); + tempCachedMap.remove(identifier); + Item item = node2Item.getItem(property.getParent(), Excludes.ALL); + insertItemInTheRightPlace(item, tempCachedMap); + } + } + break; + case Event.NODE_REMOVED: + logger.trace("node removed event received with type {}", event.getIdentifier()); + + if (tempCachedMap.get(event.getIdentifier())!=null && + tempCachedMap.get(event.getIdentifier()) tempCachedMap) throws Exception{ NodeIterator nodeIt = node.getNodes(); while(nodeIt.hasNext()) { Node child = nodeIt.nextNode(); Item item = node2Item.getItem(child, Excludes.ALL); if (item==null || item.isHidden()) continue; if (item instanceof FolderItem) - visitChildren(child); + visitChildren(child,tempCachedMap); else if(item instanceof AbstractFileItem) - insertItemInTheRightPlace(item); + insertItemInTheRightPlace(item,tempCachedMap); } } diff --git a/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java b/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java index b101740..d407da5 100644 --- a/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java @@ -345,7 +345,13 @@ public class MessageManager extends Impersonable{ NodeIterator nodeIt = node.getNodes(); while(nodeIt.hasNext()) { Node child = nodeIt.nextNode(); + log.info("message type "+child.getPrimaryNodeType().getName()); Message message = node2Item.getMessageItem(child); + if (message == null) { + log.info("message discarded"); + continue; + } + if (reduceBody != null && reduceBody>0 && message.getBody().length()>reduceBody ) message.setBody(message.getBody().substring(0, reduceBody)); insertOrdered(messages, message);