diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java index 802d33b..c3abb35 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VRE.java @@ -9,6 +9,7 @@ import javax.jcr.SimpleCredentials; import org.gcube.common.storagehub.model.exceptions.BackendGenericError; import org.gcube.common.storagehub.model.items.Item; +import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,11 +22,11 @@ public class VRE { private VREQueryRetriever vreQueryRetriever; private ExecutorService executor; - protected VRE(Item item, Repository repository, SimpleCredentials credentials, ExecutorService executor) { + protected VRE(Item item, Repository repository, SimpleCredentials credentials, Node2ItemConverter node2Item, ExecutorService executor) { super(); this.vreFolder = item; this.executor = executor; - vreQueryRetriever = new VREQueryRetriever(repository, credentials, vreFolder); + vreQueryRetriever = new VREQueryRetriever(repository, credentials, node2Item, vreFolder); result = executor.submit(vreQueryRetriever); } diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java index ebae0d0..cc1cab2 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREManager.java @@ -8,9 +8,7 @@ import java.util.concurrent.Executors; import javax.inject.Inject; import javax.inject.Singleton; -import javax.jcr.ItemNotFoundException; import javax.jcr.Node; -import javax.jcr.NodeIterator; import javax.jcr.RepositoryException; import javax.jcr.SimpleCredentials; import javax.servlet.ServletContext; @@ -76,7 +74,7 @@ public class VREManager { logger.trace("inserting VRE {}",vreFolder.getTitle()); if (vreMap.containsKey(vreFolder.getTitle())) throw new RuntimeException("something went wrong (vre already present in the map)"); else { - VRE toReturn = new VRE(vreFolder, repository.getRepository(), credentials, executor); + VRE toReturn = new VRE(vreFolder, repository.getRepository(), credentials, node2Item, executor); vreMap.put(vreFolder.getTitle(), toReturn); return toReturn; } diff --git a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java index 1a77bf8..2d37b2c 100644 --- a/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java +++ b/src/main/java/org/gcube/data/access/storagehub/handlers/vres/VREQueryRetriever.java @@ -1,29 +1,33 @@ package org.gcube.data.access.storagehub.handlers.vres; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Iterator; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; -import java.util.Locale; +import java.util.Map; import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import javax.inject.Inject; import javax.jcr.Credentials; import javax.jcr.Node; +import javax.jcr.NodeIterator; import javax.jcr.Property; import javax.jcr.Repository; import javax.jcr.RepositoryException; import javax.jcr.Session; import javax.jcr.observation.Event; import javax.jcr.observation.EventJournal; -import javax.jcr.query.Query; -import org.apache.jackrabbit.util.ISO9075; import org.gcube.common.storagehub.model.Excludes; import org.gcube.common.storagehub.model.NodeConstants; +import org.gcube.common.storagehub.model.exceptions.BackendGenericError; +import org.gcube.common.storagehub.model.items.AbstractFileItem; +import org.gcube.common.storagehub.model.items.FolderItem; import org.gcube.common.storagehub.model.items.Item; -import org.gcube.data.access.storagehub.NodeChildrenFilterIterator; import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,64 +41,32 @@ public class VREQueryRetriever implements Callable> { private Repository repository; private Credentials credentials; private Item vreFolder; - List cachedList = new ArrayList<>(CACHE_DIMENSION); + Map cachedList = new HashMap<>(CACHE_DIMENSION); + long higherTimestamp = Long.MAX_VALUE; + long lastTimestamp =0; - private Node2ItemConverter node2Item = new Node2ItemConverter(); + Node2ItemConverter node2Item; - public VREQueryRetriever(Repository repository, Credentials credentials, Item vreFolder) { + public VREQueryRetriever(Repository repository, Credentials credentials, Node2ItemConverter node2Item, Item vreFolder) { super(); this.repository = repository; this.credentials = credentials; this.vreFolder = vreFolder; + this.node2Item = node2Item; } public List call() { - logger.trace("executing recents task"); + logger.debug("executing recents task for {}",vreFolder.getTitle()); Session ses = null; - if (lastTimestamp==0) { - try { - long start = System.currentTimeMillis(); - ses = repository.login(credentials); - - Calendar now = Calendar.getInstance(); - now.add(Calendar.YEAR, -1); - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS", Locale.ENGLISH); - String formattedDate = formatter.format(now.getTime()); - - - String xpath = String.format("/jcr:root%s//element(*,nthl:workspaceLeafItem)[@jcr:lastModified>xs:dateTime('%s')] order by @jcr:lastModified descending",ISO9075.encodePath(vreFolder.getPath()),formattedDate); - - //String query = String.format("SELECT * FROM [nthl:workspaceLeafItem] AS node WHERE ISDESCENDANTNODE('%s') ORDER BY node.[jcr:lastModified] DESC ",vreFolder.getPath()); - logger.debug("query for recents is {}",xpath); - - Query jcrQuery = ses.getWorkspace().getQueryManager().createQuery(xpath, Query.XPATH); - - jcrQuery.setLimit(CACHE_DIMENSION); - lastTimestamp = System.currentTimeMillis(); - NodeChildrenFilterIterator it = new NodeChildrenFilterIterator(jcrQuery.execute().getNodes()); - logger.debug("query for recents took {}",System.currentTimeMillis()-start); - while (it.hasNext()) { - Node node = it.next(); - - Item item = node2Item.getItem(node, Excludes.EXCLUDE_ACCOUNTING); - logger.debug("RECENTS - adding item {} with timestamp {}",item.getTitle(), item.getLastModificationTime().getTimeInMillis()); - cachedList.add(item); - - } - logger.debug("creating objects took {}",System.currentTimeMillis()-start); - if (cachedList.size()<=10) return cachedList; - else return cachedList.subList(0, 10); - } catch (Exception e) { - logger.error("error querying vre {}",vreFolder.getTitle(),e); - throw new RuntimeException(e); - }finally{ - if (ses!=null) - ses.logout(); - logger.trace("recents task finished"); - } - } else { + try { + ses = repository.login(credentials); + //if (cachedList.isEmpty()) { + + init(ses); + /*} else { + logger.debug("redoing recents for {}",vreFolder.getTitle()); try { long timestampToUse = lastTimestamp; @@ -107,7 +79,7 @@ public class VREQueryRetriever implements Callable> { EventJournal journalChanged = ses.getWorkspace().getObservationManager().getEventJournal(Event.PROPERTY_CHANGED^Event.NODE_REMOVED^Event.NODE_MOVED^Event.NODE_ADDED, vreFolder.getPath(), true, null, types); journalChanged.skipTo(timestampToUse); - logger.trace("getting the journal took {}",System.currentTimeMillis()-start); + logger.debug("getting the journal took {}",System.currentTimeMillis()-start); int events = 0; @@ -122,6 +94,8 @@ public class VREQueryRetriever implements Callable> { if (nodeAdded.isNodeType("nthl:workspaceLeafItem")) { logger.trace("node added event received with name {}", nodeAdded.getName()); Item item = node2Item.getItem(nodeAdded, Arrays.asList(NodeConstants.ACCOUNTING_NAME)); + if (cachedList.get(event.getIdentifier())!=null) + cachedList.remove(event.getIdentifier()); insertItemInTheRightPlace(item); } } @@ -133,7 +107,7 @@ public class VREQueryRetriever implements Callable> { if (property.getName().equalsIgnoreCase("jcr:lastModified")) { logger.trace("event property changed on {} with value {} and parent {}",property.getName(), property.getValue().getString(), property.getParent().getPath()); String identifier = property.getParent().getIdentifier(); - cachedList.removeIf(i -> i.getId().equals(identifier)); + cachedList.remove(identifier); Item item = node2Item.getItem(property.getParent(), Excludes.EXCLUDE_ACCOUNTING); insertItemInTheRightPlace(item); } @@ -141,14 +115,10 @@ public class VREQueryRetriever implements Callable> { break; case Event.NODE_REMOVED: logger.trace("node removed event received with type {}", event.getIdentifier()); - cachedList.removeIf(i -> { - try { - return i.getId().equals(event.getIdentifier()) && i.getLastModificationTime().getTime().getTime()> { if (nodeMoved.isNodeType("nthl:workspaceLeafItem")) { logger.trace("event node moved on {} with path {}",nodeMoved.getName(), nodeMoved.getPath()); String identifier = nodeMoved.getIdentifier(); - cachedList.removeIf(i -> i.getId().equals(identifier) && !i.getPath().startsWith(vreFolder.getPath())); + String nodePath = ses.getNode(identifier).getPath(); + if (cachedList.get(event.getIdentifier())!=null && + !nodePath.startsWith(vreFolder.getPath())) + cachedList.remove(event.getIdentifier()); + } break; default: @@ -165,11 +139,9 @@ public class VREQueryRetriever implements Callable> { } - if (cachedList.size()>CACHE_DIMENSION) - cachedList.subList(51, cachedList.size()).clear(); + logger.trace("retrieving event took {} with {} events",System.currentTimeMillis()-start, events); - if (cachedList.size()<=10) return cachedList; - else return cachedList.subList(0, 10); + } catch (Exception e) { logger.error("error getting events for vre {}",vreFolder.getTitle(),e); throw new RuntimeException(e); @@ -177,20 +149,87 @@ public class VREQueryRetriever implements Callable> { if (ses!=null) ses.logout(); } + } */ + + return correctValues(ses); + } catch (Exception e) { + logger.error("error preparing recents for folder {}", vreFolder.getTitle(),e); + return Collections.emptyList(); + }finally{ + if (ses!=null) + ses.logout(); + logger.debug("recents task finished"); + } + } + + + + private List correctValues(Session ses){ + logger.debug("preparing returning values for {}",vreFolder.getTitle()); + long start = System.currentTimeMillis(); + List> list = new LinkedList<>(cachedList.entrySet()); + list.sort((c1, c2) -> c1.getValue().compareTo(c2.getValue())*-1); + if (list.size()>CACHE_DIMENSION) + for (int index = CACHE_DIMENSION-1; index< list.size() ; index++) + cachedList.remove(list.get(index).getKey()); + + List cachedIds = list.stream().map(m -> m.getKey()).collect(Collectors.toList()); + if (cachedIds.size()>10) + cachedIds = cachedIds.subList(0, 10); + + List result = new ArrayList<>(10); + + for (String id: cachedIds) { + try { + Item item = node2Item.getItem(id, ses, Excludes.EXCLUDE_ACCOUNTING); + if (item!=null) + result.add(item); + else logger.warn("item with id {} is null",id); + } catch (BackendGenericError | RepositoryException e) { } + } + + logger.debug("returning values prepared in {} for {}",System.currentTimeMillis()-start,vreFolder.getTitle()); + + return result; + } + + private void insertItemInTheRightPlace(Item item) { + long lastModifiedTime = item.getLastModificationTime().getTime().getTime(); + + if (!(lastModifiedTime>higherTimestamp && cachedList.size()>CACHE_DIMENSION)) { + cachedList.put(item.getId(), lastModifiedTime); + if (lastModifiedTime>higherTimestamp) higherTimestamp = lastModifiedTime; } } - private void insertItemInTheRightPlace(Item item) { - Iterator it = cachedList.iterator(); - int index =0; - while (it.hasNext()) { - Item inListItem = it.next(); - if (item.getLastModificationTime().getTime().getTime()>=inListItem.getLastModificationTime().getTime().getTime()) break; - index++; + private void init(Session ses){ + try { + long start = System.currentTimeMillis(); + Calendar now = Calendar.getInstance(); + now.add(Calendar.YEAR, -1); + lastTimestamp = System.currentTimeMillis(); + Node vreFolderNode = ses.getNodeByIdentifier(vreFolder.getId()); + logger.debug("starting visiting children for {}",vreFolder.getTitle()); + visitChildren(vreFolderNode); + logger.debug("initializing recents for {} took {}",vreFolder.getTitle(),System.currentTimeMillis()-start); + } catch (Exception e) { + logger.error("error querying vre {}",vreFolder.getTitle(),e); + throw new RuntimeException(e); + } + } + + private void visitChildren(Node node) throws Exception{ + NodeIterator nodeIt = node.getNodes(); + while(nodeIt.hasNext()) { + Node child = nodeIt.nextNode(); + Item item = node2Item.getItem(child, Excludes.ALL); + if (item==null || item.isHidden()) continue; + if (item instanceof FolderItem) + visitChildren(child); + else if(item instanceof AbstractFileItem) + insertItemInTheRightPlace(item); } - if (index getChildren(Predicate checker, Node parent, List excludes, boolean showHidden, Class nodeTypeToInclude) throws RepositoryException, BackendGenericError { - return Utils.getItemList(checker, parent, excludes, new Range(0, 100), showHidden, nodeTypeToInclude); + return Utils.getItemList(checker, parent, excludes, null, showHidden, nodeTypeToInclude); } @Override