From 097e1a5483a8b3d4a12ba6d492eb6273c469b817 Mon Sep 17 00:00:00 2001 From: Manuele Simi Date: Wed, 2 Mar 2011 20:52:06 +0000 Subject: [PATCH] merging from branch 3.0 git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/information-system/gCubeIS/Collector@35556 82a268e6-3cf1-43bd-a215-b396298e98cf --- .classpath | 4 +- README | 6 +- etc/deploy-jndi-config.xml | 2 +- .../porttypes/wsdaix/XMLCollectionAccess.java | 7 +- .../impl/resources/BaseDAIXResource.java | 6 +- .../impl/resources/GCUBEXMLResource.java | 114 +++++++++++++----- .../state/AggregatorRegisteredResource.java | 4 +- .../collector/impl/utils/MetadataReader.java | 50 ++++++-- .../impl/xmlstorage/exist/State.java | 1 + .../impl/xmlstorage/exist/XMLStorage.java | 8 ++ .../exist/sweep/ResourceFilter.java | 87 +++++++++++++ .../xmlstorage/exist/{ => sweep}/Sweeper.java | 8 +- .../exist/sweep/XQuery-RIStatus.xml | 4 + .../stubs/metadata/MetadataRecord.java | 34 +++++- .../stubs/metadata/MetadataWriter.java | 5 +- .../testsuite/wsdaix/AddDocumentsTester.java | 2 +- test-suite/samples/xquery-RI.xml | 4 + .../collector/XMLStorageManagerTest.java | 1 - 18 files changed, 286 insertions(+), 61 deletions(-) create mode 100644 src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/ResourceFilter.java rename src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/{ => sweep}/Sweeper.java (95%) create mode 100644 src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/XQuery-RIStatus.xml create mode 100644 test-suite/samples/xquery-RI.xml diff --git a/.classpath b/.classpath index 9764692..38f2242 100644 --- a/.classpath +++ b/.classpath @@ -7,8 +7,8 @@ - - + + diff --git a/README b/README index 7eb44e0..f582257 100755 --- a/README +++ b/README @@ -16,7 +16,7 @@ Authors Version and Release Date ------------------------ -v. 2.2.0 (30-11-2009) +v. 3.0.0 (07-02-2011) Description ----------- @@ -29,14 +29,14 @@ Source code is available from SVN: http://svn.d4science.research-infrastructures.eu/gcube/trunk/ Binaries can be downloaded from: -http://software.d4science.research-infrastructures.eu/ +https://www.gcube-system.org/index.php?option=com_content&view=article&id=290&Itemid=48 Documentation ------------- Documentation is available on-line from the Projects Documentation Wiki: -https://technical.wiki.d4science.research-infrastructures.eu/documentation/index.php/IS-Collector +https://gcube.wiki.gcube-system.org/gcube/index.php/IS-Collector Licensing --------- diff --git a/etc/deploy-jndi-config.xml b/etc/deploy-jndi-config.xml index cf11c89..fdb7cc2 100755 --- a/etc/deploy-jndi-config.xml +++ b/etc/deploy-jndi-config.xml @@ -26,7 +26,7 @@ - //2 - @@ -200,7 +201,9 @@ public class XMLCollectionAccess extends GCUBEPortType { } try { + //logger.trace("resource to wrap " + resource.toString()); GCUBEXMLResource xmlResource = wrap(resource, metadataReader); + //logger.trace("wrapped resource " + xmlResource.toString()); boolean exist = false; if ( State.getDataManager().resourceExists(xmlResource) ) { exist = true; @@ -267,6 +270,8 @@ public class XMLCollectionAccess extends GCUBEPortType { xmlResource.setGroupKey(metadataReader.getGroupKey()); xmlResource.setEntryKey(metadataReader.getEntryKey()); xmlResource.setSourceKey(metadataReader.getKey()); + xmlResource.setNamespace(metadataReader.getNamespace()); + xmlResource.setPublicationMode(metadataReader.getPublicationMode()); } return xmlResource; } diff --git a/src/org/gcube/informationsystem/collector/impl/resources/BaseDAIXResource.java b/src/org/gcube/informationsystem/collector/impl/resources/BaseDAIXResource.java index f04585a..e880600 100644 --- a/src/org/gcube/informationsystem/collector/impl/resources/BaseDAIXResource.java +++ b/src/org/gcube/informationsystem/collector/impl/resources/BaseDAIXResource.java @@ -102,6 +102,7 @@ public class BaseDAIXResource implements DAIXResource { } else if (this.dataAsString.compareToIgnoreCase("") != 0) { try { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(false); DocumentBuilder builder = factory.newDocumentBuilder(); StringReader reader = new StringReader(this.dataAsString); InputSource source = new InputSource(reader); @@ -168,6 +169,7 @@ public class BaseDAIXResource implements DAIXResource { return this.toString(); Node targetNode = nodelist.item(0); TransformerFactory transFactory = TransformerFactory.newInstance(); + Transformer transformer = transFactory.newTransformer(); StringBuilder ret = new StringBuilder(); transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); @@ -177,8 +179,8 @@ public class BaseDAIXResource implements DAIXResource { StringWriter buffer = new StringWriter(); transformer.transform(new DOMSource(node), new StreamResult(buffer)); ret.append(buffer.toString().trim()); - node = targetNode.getChildNodes().item(index++); - } + node = targetNode.getChildNodes().item(++index); + } return ret.toString(); } catch (Exception e) { diff --git a/src/org/gcube/informationsystem/collector/impl/resources/GCUBEXMLResource.java b/src/org/gcube/informationsystem/collector/impl/resources/GCUBEXMLResource.java index 5849250..93b257f 100644 --- a/src/org/gcube/informationsystem/collector/impl/resources/GCUBEXMLResource.java +++ b/src/org/gcube/informationsystem/collector/impl/resources/GCUBEXMLResource.java @@ -2,8 +2,10 @@ package org.gcube.informationsystem.collector.impl.resources; import java.io.StringReader; import java.io.StringWriter; +import java.util.ArrayList; import java.util.Calendar; import java.util.GregorianCalendar; +import java.util.List; import java.util.TimeZone; import javax.xml.parsers.DocumentBuilder; @@ -14,13 +16,16 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; +import org.gcube.common.core.state.GCUBEWSResourcePropertySet; import org.gcube.common.core.utils.logging.GCUBELog; import org.gcube.informationsystem.collector.impl.resources.DAIXResource.MalformedResourceException; import org.w3c.dom.Document; import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import org.xml.sax.InputSource; @@ -45,13 +50,17 @@ public class GCUBEXMLResource { protected Calendar terminationTime = null, lastUpdateTime = null; - protected String entryKey, groupKey, source, sourceKey = "", completeSourceKey = ""; + protected String entryKey, groupKey, source, sourceKey = "", namespace = ""; // xpath factory to evaluate Xpath expressions - private XPath path = XPathFactory.newInstance().newXPath(); + static protected XPath path = XPathFactory.newInstance().newXPath(); + + static protected Transformer transformer; private DAIXResource resource; + private String publicationMode = ""; + public GCUBEXMLResource(DAIXResource resource) throws MalformedXMLResourceException { this.resource = resource; this.terminationTime = new GregorianCalendar(); @@ -185,6 +194,15 @@ public class GCUBEXMLResource { public String getSource() { return this.source; } + + /** + * The mode in which the resource was published + * + * @return the mode + */ + public String getPublicationMode() { + return this.publicationMode; + } /** * Sets the key of the WS-Resource that published the data @@ -206,24 +224,28 @@ public class GCUBEXMLResource { } /** - * Sets the complete source key + * Sets the namespace * - * @param completeKey - * the new complete key + * @param namamespace + * the namespace */ - public void setCompleteSourceKey(String completeKey) { - this.completeSourceKey = completeKey; + public void setNamespace(String namespace) { + this.namespace = namespace; } /** - * Gets the complete source key + * Gets the namespace * - * @return the complete source key + * @return the namespace */ - public String getCompleteSourceKey() { - return this.completeSourceKey; + public String getNamespace() { + return this.namespace; } + public void setPublicationMode(String publicationMode) { + this.publicationMode = publicationMode; + + } /** * Sets the content of the resource * @param content the content @@ -233,6 +255,7 @@ public class GCUBEXMLResource { try { if (enveloped != null && enveloped.length > 0 && enveloped[0]) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(false); DocumentBuilder builder = factory.newDocumentBuilder(); StringReader reader = new StringReader(content); InputSource source = new InputSource(reader); @@ -263,11 +286,15 @@ public class GCUBEXMLResource { value = path.evaluate("Document/Source", doc); this.setSource(value); value = path.evaluate("Document/SourceKey", doc); - this.setCompleteSourceKey(value); + this.setSourceKey(value); value = path.evaluate("Document/EntryKey", doc); this.setEntryKey(value); value = path.evaluate("Document/GroupKey", doc); this.setGroupKey(value); + value = path.evaluate("Document/Namespace", doc); + this.setNamespace(value); + value = path.evaluate("Document/PublicationMode", doc); + this.setPublicationMode(value); } catch (Exception xpee) { logger.error("" + xpee.getMessage()); logger.error("" + xpee.getStackTrace()); @@ -280,16 +307,26 @@ public class GCUBEXMLResource { // we do not use an XML parser for performance reasons StringBuilder resource = new StringBuilder("\n"); try { + resource.append("" + this.resource.getResourceName() + "\n"); resource.append("" + this.getSource() + "\n"); resource.append("" + this.getSourceKey() + "\n"); - resource.append("" + this.getCompleteSourceKey() + "\n"); + + if (this.getNamespace()!=null && this.getNamespace().length() > 0) { + String completeKey = "KEY"; + completeKey = completeKey.replace("NS", this.getNamespace()); + completeKey = completeKey.replace("KEY", this.getSourceKey()); + resource.append("" + completeKey + "\n"); + } else { + resource.append("\n"); + } resource.append("" + this.getEntryKey() + "\n"); resource.append("" + this.getGroupKey() + "\n"); resource.append("" + this.getTerminationTime().getTimeInMillis() + "\n"); resource.append("" + this.getTerminationTime().getTime().toString() + "\n"); resource.append("" + this.lastUpdateTime.getTimeInMillis() + "\n"); resource.append("" + this.lastUpdateTime.getTime().toString() + "\n"); + resource.append("" + this.publicationMode + "\n"); resource.append("\n"); if (this.resource instanceof GCUBEInstanceStateResource) { //this check is to avoid a stupid message ("[Fatal Error] :1:83: The markup in the document following the root element must be well-formed.") in the nohup.out from the sax parser @@ -378,6 +415,7 @@ public class GCUBEXMLResource { resource.append(this.resource.toString() + "\n"); resource.append("\n"); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(false); DocumentBuilder builder = factory.newDocumentBuilder(); StringReader reader = new StringReader(resource.toString()); InputSource source = new InputSource(reader); @@ -396,20 +434,20 @@ public class GCUBEXMLResource { * @param content the content to trim * @return the trimmed content */ - private String trimProperties(final String content) { - String trimmedContent = content; - String elem = "<"+GCUBEInstanceStateResource.INSTANCESTATE_ROOT_ELEMENT +">"; - if (content.startsWith(elem)) { - trimmedContent=trimmedContent.substring(elem.length(), trimmedContent.length()); - } - - elem = ""; - if (content.endsWith(elem)) { - //trimmedContent=trimmedContent.replace(, ""); - trimmedContent=trimmedContent.substring(0, trimmedContent.length() - elem.length()); - } - return trimmedContent; - } +// private String trimProperties(final String content) { +// String trimmedContent = content; +// String elem = "<"+GCUBEInstanceStateResource.INSTANCESTATE_ROOT_ELEMENT +">"; +// if (content.startsWith(elem)) { +// trimmedContent=trimmedContent.substring(elem.length(), trimmedContent.length()); +// } +// +// elem = ""; +// if (content.endsWith(elem)) { +// //trimmedContent=trimmedContent.replace(, ""); +// trimmedContent=trimmedContent.substring(0, trimmedContent.length() - elem.length()); +// } +// return trimmedContent; +// } /** * @@ -430,5 +468,27 @@ public class GCUBEXMLResource { } } + public String getSourceRunningInstance() throws Exception { + try { + + if (transformer==null) { + transformer = TransformerFactory.newInstance().newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION,"yes"); + } + List results = new ArrayList(); + NodeList set = (NodeList) path.evaluate("/"+GCUBEWSResourcePropertySet.RP_RIID_NAME+"/text()", + this.resource.getContent(), XPathConstants.NODESET); + for (int i=0;i resource.getTerminationTime().getTimeInMillis()) + return true; + else + return false; + } + + private static String readInputStreamAsString(InputStream in) throws IOException { + BufferedInputStream bis = new BufferedInputStream(in); + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + int result = bis.read(); + while(result != -1) { + byte b = (byte)result; + buf.write(b); + result = bis.read(); + } + return buf.toString(); + } + + private static String getRIStatus(String id)throws Exception { + InputStream statusquery = ResourceFilter.class.getResourceAsStream("XQuery-RIStatus.xml"); + XQuery q = new XQuery(readInputStreamAsString(statusquery).replace("$1",id)); + ResourceSet result = State.getQueryManager().executeXQuery(q); + XMLResource xmlres = (XMLResource) result.getResource((long) 0); + return ((String) xmlres.getContent()).trim(); + } + +} diff --git a/src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/Sweeper.java b/src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/Sweeper.java similarity index 95% rename from src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/Sweeper.java rename to src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/Sweeper.java index b1e99c9..3ab22ee 100755 --- a/src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/Sweeper.java +++ b/src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/Sweeper.java @@ -1,7 +1,7 @@ /** * */ -package org.gcube.informationsystem.collector.impl.xmlstorage.exist; +package org.gcube.informationsystem.collector.impl.xmlstorage.exist.sweep; import java.lang.InterruptedException; import java.util.GregorianCalendar; @@ -12,8 +12,8 @@ import java.util.List; import org.gcube.common.core.utils.logging.GCUBELog; import org.gcube.informationsystem.collector.impl.xmlstorage.exist.State; -import org.gcube.informationsystem.collector.impl.xmlstorage.exist.Sweeper; import org.gcube.informationsystem.collector.impl.xmlstorage.exist.XMLStorage.XMLStorageNotAvailableException; +import org.gcube.informationsystem.collector.impl.xmlstorage.exist.sweep.Sweeper; import org.gcube.informationsystem.collector.impl.resources.GCUBEInstanceStateResource; import org.gcube.informationsystem.collector.impl.resources.GCUBEXMLResource; @@ -86,9 +86,7 @@ public class Sweeper implements Runnable { GCUBEXMLResource xmlresource = new GCUBEXMLResource(tempresource); State.getDataManager().retrieveResourceContent(xmlresource); logger.trace("Checking resource " + id); - logger.trace("Now is " + now.getTimeInMillis()); - logger.trace("Resource expires at " + xmlresource.getTerminationTime().getTimeInMillis()); - if (now.getTimeInMillis() > xmlresource.getTerminationTime().getTimeInMillis()) { + if (ResourceFilter.isExpired(xmlresource)) { // removes the resources from the database State.getDataManager().deleteResource(xmlresource); logger.info("Resource " + xmlresource.getResourceName() + " deleted"); diff --git a/src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/XQuery-RIStatus.xml b/src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/XQuery-RIStatus.xml new file mode 100644 index 0000000..5fe9259 --- /dev/null +++ b/src/org/gcube/informationsystem/collector/impl/xmlstorage/exist/sweep/XQuery-RIStatus.xml @@ -0,0 +1,4 @@ +for $RIs in collection("/db/Profiles/RunningInstance")//Document/Data/child::*[local-name()='Profile']/Resource +where $RIs/ID eq '$1' +return + $RIs/Profile/DeploymentData/Status/string() diff --git a/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataRecord.java b/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataRecord.java index 88ea011..be31ad7 100644 --- a/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataRecord.java +++ b/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataRecord.java @@ -30,6 +30,8 @@ public final class MetadataRecord { }; + static GCUBELog logger = new GCUBELog(MetadataRecord.class); + private Calendar terminationTime; private Integer time; @@ -37,6 +39,10 @@ public final class MetadataRecord { private String source, key, groupKey, entryKey; private TYPE type; + + private String publicationMode = ""; + + private String namespace = ""; protected MetadataRecord() {} @@ -147,8 +153,10 @@ public final class MetadataRecord { */ public Document getAsDocument() throws SAXException, IOException, ParserConfigurationException { - return DocumentBuilderFactory - .newInstance() + DocumentBuilderFactory factory = DocumentBuilderFactory + .newInstance(); + factory.setNamespaceAware(false); + return factory .newDocumentBuilder() .parse(new ByteArrayInputStream(getAsBuilder().toString().getBytes("UTF-8"))); } @@ -173,8 +181,30 @@ public final class MetadataRecord { builder.append("").append(this.getTimeToLive()).append(""); builder.append("").append(this.getGroupKey()).append(""); builder.append("").append(this.getEntryKey()).append(""); + builder.append("").append(this.getNamespace()).append(""); builder.append("").append(this.getKey()).append(""); + builder.append("").append(this.getPublicationMode()).append(""); builder.append(""); + //logger.info("Metadata document " + builder.toString()); return builder; } + + private Object getNamespace() { + return this.namespace; + } + + public String getPublicationMode() { + return this.publicationMode; + } + + public void setPublicationMode(String publicationMode) { + this.publicationMode = publicationMode; + + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + + } + } diff --git a/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataWriter.java b/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataWriter.java index 7cc4a51..1726c40 100644 --- a/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataWriter.java +++ b/src/org/gcube/informationsystem/collector/stubs/metadata/MetadataWriter.java @@ -23,9 +23,10 @@ public final class MetadataWriter { * @param groupkey * @param key * @param entrykey + * @param publicationMode the mode in which the resource is published */ public MetadataWriter(TYPE type, String source, - Integer timeToLive, String groupkey, String key, String entrykey) { + Integer timeToLive, String groupkey, String key, String entrykey, String namespace, String publicationMode) { this.metadata = new MetadataRecord(); this.metadata.setType(type); this.metadata.setSource(source); @@ -33,6 +34,8 @@ public final class MetadataWriter { this.metadata.setGroupKey(groupkey); this.metadata.setKey(key); this.metadata.setTimeToLive(timeToLive); + this.metadata.setPublicationMode(publicationMode); + this.metadata.setNamespace(namespace); } public MetadataRecord getRecord() { diff --git a/src/org/gcube/informationsystem/collector/stubs/testsuite/wsdaix/AddDocumentsTester.java b/src/org/gcube/informationsystem/collector/stubs/testsuite/wsdaix/AddDocumentsTester.java index 7c91df4..0518947 100644 --- a/src/org/gcube/informationsystem/collector/stubs/testsuite/wsdaix/AddDocumentsTester.java +++ b/src/org/gcube/informationsystem/collector/stubs/testsuite/wsdaix/AddDocumentsTester.java @@ -97,7 +97,7 @@ public class AddDocumentsTester { MessageElement msgElement = new MessageElement(documents[i].getDocumentElement()); MessageElement msgElement2; MetadataWriter writer = new MetadataWriter(TYPE.INSTANCESTATE, "http://source", - 600, "MyGroupKey", "MyKey", "MyEntryKey"); + 600, "MyGroupKey", "MyKey", "MyEntryKey","","push"); try { msgElement2 = new MessageElement(writer.getRecord().getAsDocument().getDocumentElement()); } catch (Exception e) { diff --git a/test-suite/samples/xquery-RI.xml b/test-suite/samples/xquery-RI.xml new file mode 100644 index 0000000..37d5b3d --- /dev/null +++ b/test-suite/samples/xquery-RI.xml @@ -0,0 +1,4 @@ +for $RIs in collection("/db/Profiles/RunningInstance")//Document/Data/child::*[local-name()='Profile']/Resource +where $RIs/ID eq '3037b1c0-180e-11e0-b6c4-d7845129cab1' +return + $RIs/Profile/DeploymentData/Status/string() diff --git a/test/org/gcube/informationsystem/collector/XMLStorageManagerTest.java b/test/org/gcube/informationsystem/collector/XMLStorageManagerTest.java index 3fce190..a5e0b16 100644 --- a/test/org/gcube/informationsystem/collector/XMLStorageManagerTest.java +++ b/test/org/gcube/informationsystem/collector/XMLStorageManagerTest.java @@ -1,6 +1,5 @@ package org.gcube.informationsystem.collector; -import ise.antelope.tasks.util.Base64.InputStream; import java.io.BufferedReader; import java.io.IOException;