From 4b04fd72248fed526f7ab1e6adbf58b75498d18d Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Thu, 19 Sep 2019 10:30:07 +0000 Subject: [PATCH] Merged git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/sis-geotk-plugin@181739 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 6 + pom.xml | 20 +- .../ApplicationConfigurationRetriever.java | 43 ++ .../plugins/thredds/CatalogDescriptor.java | 19 + .../transfer/plugins/thredds/CommonXML.java | 133 ++++++ .../plugins/thredds/LocalConfiguration.java | 42 ++ .../thredds/ThreddsInstanceManager.java | 394 ++++++++++++++++++ .../thredds/TomcatSecurityHandler.java | 29 ++ .../plugins/thredds/XMLCatalogHandler.java | 193 +++++++++ .../catalog/RegisterCatalogPlugin.java | 54 +++ .../catalog/RegisterCatalogPluginFactory.java | 103 +++++ .../{ => thredds}/sis/SISPluginFactory.java | 66 ++- .../plugins/{ => thredds}/sis/SisPlugin.java | 105 ++--- ...data.transfer.plugin.AbstractPluginFactory | 3 +- src/main/resources/thredds.properties | 9 + .../transfer/plugins/sis/TestAddCatalog.java | 21 + .../transfer/plugins/sis/TestGetCatalog.java | 27 +- .../transfer/plugins/sis/TestGetMetadata.java | 1 + 18 files changed, 1151 insertions(+), 117 deletions(-) create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/ApplicationConfigurationRetriever.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/CatalogDescriptor.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/CommonXML.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/LocalConfiguration.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/TomcatSecurityHandler.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/XMLCatalogHandler.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/catalog/RegisterCatalogPlugin.java create mode 100644 src/main/java/org/gcube/data/transfer/plugins/thredds/catalog/RegisterCatalogPluginFactory.java rename src/main/java/org/gcube/data/transfer/plugins/{ => thredds}/sis/SISPluginFactory.java (72%) rename src/main/java/org/gcube/data/transfer/plugins/{ => thredds}/sis/SisPlugin.java (50%) create mode 100644 src/main/resources/thredds.properties create mode 100644 src/test/java/org/gcube/data/transfer/plugins/sis/TestAddCatalog.java diff --git a/distro/changelog.xml b/distro/changelog.xml index eb8fc6c..b8d8a6f 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,5 +1,11 @@ First Release + + + Bug fix + + + RegisterCatalogPlugin \ No newline at end of file diff --git a/pom.xml b/pom.xml index 33acdd9..9b809f7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,11 +4,11 @@ org.gcube.tools maven-parent - LATEST + 1.1.0-SNAPSHOT org.gcube.data.transfer sis-geotk-plugin - 1.0.1-SNAPSHOT + 1.1.1-SNAPSHOT Sis/GeoToolkit plugin Apache Sis/Geotk plugin for data-transfer-service @@ -52,6 +52,15 @@ provided + + org.gcube.resources + registry-publisher + + + org.gcube.core + common-encryption + + @@ -101,7 +110,12 @@ junit test - + + org.gcube.data.transfer + data-transfer-library + [1.2.0-SNAPSHOT,2.0.0-SNAPSHOT) + test + diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/ApplicationConfigurationRetriever.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/ApplicationConfigurationRetriever.java new file mode 100644 index 0000000..6dce768 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/ApplicationConfigurationRetriever.java @@ -0,0 +1,43 @@ +package org.gcube.data.transfer.plugins.thredds; + +import java.util.concurrent.Callable; + +import org.gcube.data.transfer.plugin.model.DataTransferContext; +import org.gcube.smartgears.configuration.application.ApplicationConfiguration; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ApplicationConfigurationRetriever implements Callable{ + + private DataTransferContext ctx; + + private long timeout=LocalConfiguration.getTTL(LocalConfiguration.CONTEXT_LOADING_TIMETOUT); + + public ApplicationConfigurationRetriever(DataTransferContext ctx) { + this.ctx=ctx; + } + + + @Override + public ApplicationConfiguration call() throws Exception { + ApplicationConfiguration toReturn=null; + log.info("Waiting for thredds application to be loaded"); + long startTime=System.currentTimeMillis(); + while(toReturn==null&(System.currentTimeMillis()-startTime namespaces=new HashMap(); + + + + + static{ + try{ + DocumentBuilderFactory factory=DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + docBuilder = factory.newDocumentBuilder(); + + + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + transformer = transformerFactory.newTransformer(); + + + namespaces.put("xmlns", "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0"); + namespaces.put("xlink", "http://www.w3.org/1999/xlink"); + namespaces.put("xsi", "http://www.w3.org/2001/XMLSchema-instance"); + // namespaces.put("gml", "http://www.opengis.net/gml"); + // namespaces.put("xsi", "http://www.w3.org/2001/XMLSchema-instance"); + // namespaces.put("gmi", "http://www.isotc211.org/2005/gmi"); + // namespaces.put("gmx", "http://www.isotc211.org/2005/gmx"); + + + + + }catch(Exception e){ + throw new RuntimeException("Unable to init Fixer ",e); + } + } + + + public static Document getDocument(File xmlFile) throws SAXException, IOException { + log.debug("Parsing {} ",xmlFile.getAbsolutePath()); + InputSource inputSource = new InputSource(new FileInputStream(xmlFile)); + //Get document owner + Element documentNode = docBuilder.parse(inputSource).getDocumentElement(); + return documentNode.getOwnerDocument(); + } + + public static XPathHelper getHelper(Node root){ + XPathHelper toReturn =new XPathHelper(root); + for(Entry entry:namespaces.entrySet()) + toReturn.addNamespace(entry.getKey(), entry.getValue()); + return toReturn; + } + + public static void writeOut(Document document, File destination) throws IOException, TransformerException { + DOMSource source = new DOMSource(document); + StreamResult result = new StreamResult(new FileWriter(destination)); + transformer.transform(source, result); + result.getWriter().flush(); + result.getWriter().close(); + } + + + public static enum Position{ + sibling_after,sibling_before,first_child,last_child,replace + } + + public static void addContent(String path, Document doc, String toAddContent, XPathHelper documentHelper,Position position) throws SAXException, IOException{ + NodeList nodelist=documentHelper.evaluateForNodes(path); + if(nodelist==null||nodelist.getLength()==0) throw new RuntimeException("Path "+path+" not found in document"); + // if(nodelist.getLength()>1) throw new RuntimeException("Invalid Path "+path+"."+nodelist.getLength()+" entries found"); + Node targetNode=nodelist.item(0); + + Document online=docBuilder.parse(new ByteArrayInputStream(toAddContent.getBytes())); + Node toAdd=doc.importNode(online.getDocumentElement(), true); + switch(position){ + case first_child: { + targetNode.insertBefore(toAdd, targetNode.getFirstChild()); + break; + } + case last_child:{targetNode.appendChild(toAdd); + break;} + case replace : { + Node parent=targetNode.getParentNode(); + parent.replaceChild(toAdd, targetNode); + break; + } + case sibling_after :{ + Node currentlyNext=targetNode.getNextSibling(); + Node parent=targetNode.getParentNode(); + if(currentlyNext!=null)parent.insertBefore(toAdd, currentlyNext); + else parent.appendChild(toAdd); + break; + } + case sibling_before :{ + Node parent=targetNode.getParentNode(); + parent.insertBefore(toAdd, targetNode); + break; + } + } + + } +} diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/LocalConfiguration.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/LocalConfiguration.java new file mode 100644 index 0000000..6bef512 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/LocalConfiguration.java @@ -0,0 +1,42 @@ +package org.gcube.data.transfer.plugins.thredds; + +import java.util.Properties; + +public class LocalConfiguration { + + final static public String THREDDS_SE_CATEGORY="th.se.category"; + final static public String THREDDS_SE_PLATFORM="th.se.platform"; + final static public String THREDDS_SE_REMOTE_MANAGEMENT_ACCESS="th.se.remoteManagement.access"; + + final static public String IS_REGISTRATION_TIMEOUT="is.registration.timeout"; + + final static public String CONTEXT_LOADING_TIMETOUT="context.loading.timeout"; + + static LocalConfiguration instance=null; + + + public static synchronized LocalConfiguration get(){ + if(instance==null) + instance=new LocalConfiguration(); + return instance; + } + + private Properties props=new Properties(); + + private LocalConfiguration() { + try{ + props.load(this.getClass().getResource("/thredds.properties").openStream()); + }catch(Exception e){ + throw new RuntimeException(e); + } + } + + public static String getProperty(String property){ + return get().props.getProperty(property); + } + + public static Long getTTL(String property) { + return Long.parseLong(getProperty(property)); + } + +} diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java new file mode 100644 index 0000000..f7c8daa --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/ThreddsInstanceManager.java @@ -0,0 +1,394 @@ +package org.gcube.data.transfer.plugins.thredds; + +import static org.gcube.resources.discovery.icclient.ICFactory.client; +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.encryption.StringEncrypter; +import org.gcube.common.resources.gcore.Resource; +import org.gcube.common.resources.gcore.Resources; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Profile; +import org.gcube.common.resources.gcore.common.Platform; +import org.gcube.common.resources.gcore.utils.Group; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.data.transfer.plugin.model.DataTransferContext; +import org.gcube.informationsystem.publisher.RegistryPublisher; +import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.resources.discovery.client.queries.impl.QueryBox; +import org.gcube.smartgears.configuration.application.ApplicationConfiguration; +import org.xml.sax.SAXException; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class ThreddsInstanceManager { + + protected static ThreddsInstanceManager instance=null; + + protected static final ConcurrentSkipListSet checkedTokens=new ConcurrentSkipListSet<>(); + + + @Synchronized + public static ThreddsInstanceManager get(DataTransferContext ctx) { + if(instance==null) + instance= new ThreddsInstanceManager(ctx); + return instance; + } + + // ***************** INSTANCE LOGIC + + + protected ThreddsInfo cachedInfo=null; + protected ExecutorService executor=Executors.newSingleThreadExecutor(); + //loaded at construction time + protected DataTransferContext ctx=null; + protected ApplicationConfiguration threddsConfig=null; + protected String threddsAdminUser; + protected String threddsAdminPassword; + protected String threddsPersistenceLocation; + protected String threddsVersionString; + protected String currentHostname; + protected String currentGHNId; + + protected ThreddsInstanceManager(DataTransferContext context) { + log.warn("Instance Creation. Should happen only once. Loading information from context.."); + this.ctx=context; + currentHostname=ctx.getCtx().container().configuration().hostname(); + currentGHNId=ctx.getCtx().container().id(); + String tomcatSecurityPath=System.getenv("WEB_CONTAINER_HOME")+"/conf/tomcat-users.xml"; + + log.info("Loading security from {} ",tomcatSecurityPath); + try{ + TomcatSecurityHandler tomcatHandler=new TomcatSecurityHandler(tomcatSecurityPath); + threddsAdminUser=tomcatHandler.getThreddsAdminUser(); + threddsAdminPassword=tomcatHandler.getThreddsAdminPassword(); + }catch(Exception e) { + throw new RuntimeException("Unable to parse security file "+tomcatSecurityPath,e); + } + log.info("Looking for Thredds Application Configuration.. "); + + + //Use Future + Future future=executor.submit(new ApplicationConfigurationRetriever(ctx)); + try { + threddsConfig=future.get(); + if(threddsConfig==null) throw new Exception("Returned Application Configuration is null"); + threddsPersistenceLocation=threddsConfig.persistence().location(); + threddsVersionString=threddsConfig.version(); + }catch(Exception e) { + throw new RuntimeException("Unable to find Application Configuration for thredds.",e); + } + + } + + /** + * + * + * @return + * @throws IOException + * @throws SAXException + * @throws Exception + */ + public synchronized ThreddsInfo getInfo() throws SAXException, IOException { + if(cachedInfo==null) { + log.info("Loading ThreddsInfo.."); + + String threddsContentRoot=getContentRoot(); + + + log.debug("Found content root at {} ",threddsContentRoot); + + //Host + ThreddsInfo info=new ThreddsInfo(); + info.setHostname(currentHostname); + info.setGhnId(currentGHNId); + info.setLocalBasePath(threddsPersistenceLocation); + info.setInstanceBaseUrl("http://"+info.getHostname()+"/thredds"); + + + String mainCatalogPath=threddsContentRoot+"/catalog.xml"; + + log.info("Loading catalog information from {} ",mainCatalogPath); + + XMLCatalogHandler handler=new XMLCatalogHandler(new File(mainCatalogPath)); + info.setCatalog(handler.getCatalogDescriptor()); + + + //tomcat security + info.setAdminPassword(threddsAdminPassword); + info.setAdminUser(threddsAdminUser); + + //version + String[] splittedVersion=threddsVersionString.split("\\."); + info.setVersion(Integer.parseInt(splittedVersion[0])); + info.setMinor(Integer.parseInt(splittedVersion[1])); + info.setRevision(Integer.parseInt(splittedVersion[2])); + log.info("Loaded ThreddsInfo is {} ",info); + cachedInfo=info; + } + return cachedInfo; + } + + public synchronized void clearCache() { + log.debug("Clearing cache.."); + cachedInfo=null; + } + + public String getCurrentHostname() { + return currentHostname; + } + + public String getMainCatalogFile(){ + return getContentRoot()+"/catalog.xml"; + } + + public String getContentRoot() { + return threddsPersistenceLocation; + } + + + public XMLCatalogHandler mainCatalogHandler() throws SAXException, IOException, Exception { + return new XMLCatalogHandler(new File(getMainCatalogFile())); + } + + @Synchronized + public void updatePublishedInfo() throws Exception { + String token=SecurityTokenProvider.instance.get(); + log.info("Checking IS with token {} ",token); + if(!checkedTokens.contains(token)) { + checkedTokens.add(token); + getInfo(); + String currentHostname=cachedInfo.getHostname(); + + log.info("Checking IS Information, host is {}",currentHostname); + + List currentEndpoints=queryForServiceEndpoints(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_CATEGORY), + LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_PLATFORM)); + + ServiceEndpoint toCheck=null; + + //Checking by host + log.debug("Found {} Service Endpoints, checking by hostname {} ",currentEndpoints.size()); + for(ServiceEndpoint se:currentEndpoints) { + String host=se.profile().runtime().hostedOn(); + try{ + if(isSameHost(host, currentHostname)) {toCheck=se; + break;} + }catch(Throwable t) { + log.warn("Unable to check Host {} ",host,t); + } + } + + if(toCheck==null) { + log.info("ServiceEndpoint not found, going to create one.."); + // CREATE NEW + ServiceEndpoint newSE=getNewServiceEndpoint(); + updateAndWait(newSE, true); + + }else { + // Check found + boolean updateSE=true; + String adminAPName=LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_REMOTE_MANAGEMENT_ACCESS); + log.debug("Looking for Access Point {} ",adminAPName); + Group existentAP=toCheck.profile().accessPoints(); + + + boolean addAccessPoint=true; + for(AccessPoint ap:existentAP) + if(ap.name().equals(adminAPName)) { + addAccessPoint=false; + // FOUND AP + String pwd=decryptString(ap.password()); + if(ap.username().equalsIgnoreCase(threddsAdminUser)&&pwd.equalsIgnoreCase(threddsAdminPassword)) { + log.info("ServiceEndopint is up to date."); + updateSE=false; + }else { + // AP is not up to date + ap.credentials(threddsAdminPassword, threddsAdminUser); + } + } + + if(updateSE) { + log.debug("Need to update SE... "); + if(addAccessPoint) { + log.debug("Access point {} not found. Adding it.. ",adminAPName); + existentAP.add(getNewAccessPoint()); + } + ServiceEndpoint updated=updateAndWait(toCheck,false); + log.info("Updated {} ",updated); + } + + } + }else log.info("Skipping token {}, already checked.",token); + + } + + private static String registerServiceEndpoint(ServiceEndpoint toRegister) { + RegistryPublisher rp=RegistryPublisherFactory.create(); + Resource r=rp.create(toRegister); + return r.id(); + } + + public static ServiceEndpoint update(ServiceEndpoint toUpdate) { + RegistryPublisher rp=RegistryPublisherFactory.create(); + return rp.update(toUpdate); + } + + private ServiceEndpoint getNewServiceEndpoint() { + ServiceEndpoint toReturn=new ServiceEndpoint(); + + Profile profile=toReturn.newProfile(); + profile.category(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_CATEGORY)); + profile.name("Thredds on "+cachedInfo.getHostname()); + profile.description("Thredds on "+cachedInfo.getHostname()); + + // TODO Gather info on version + Platform platform=profile.newPlatform(); + platform.version((short)cachedInfo.getVersion()); + platform.minorVersion((short)cachedInfo.getMinor()); + platform.revisionVersion((short)cachedInfo.getRevision()); + platform.buildVersion((short)cachedInfo.getBuild()); + platform.name(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_PLATFORM)); + + org.gcube.common.resources.gcore.ServiceEndpoint.Runtime runtime=profile.newRuntime(); + runtime.ghnId(cachedInfo.getGhnId()); + runtime.hostedOn(cachedInfo.getHostname()); + runtime.status("READY"); + + profile.accessPoints().add(getNewAccessPoint()); + return toReturn; + + } + + private static ServiceEndpoint updateAndWait(ServiceEndpoint toUpdate,boolean isNew) { + boolean equals=true; + boolean timeoutReached=false; + long timeout=LocalConfiguration.getTTL(LocalConfiguration.IS_REGISTRATION_TIMEOUT); + log.info("Going to register {}. Timeout is {} ",toUpdate.id(),timeout); + String toUpdateString=marshal(toUpdate); + log.debug("Serialized resource is {} ",toUpdateString); + if(isNew) registerServiceEndpoint(toUpdate); + else update(toUpdate); + long updateTime=System.currentTimeMillis(); + String updatedString=null; + do { + try { + Thread.sleep(500); + } catch (InterruptedException e) {} + List byIdResults=queryById(toUpdate.id()); + if(byIdResults.isEmpty()) { + equals=false; + }else { + updatedString=byIdResults.get(0); + equals=toUpdateString.equals(updatedString); + } + timeoutReached=(System.currentTimeMillis()-updateTime)>timeout; + }while(equals&&(!timeoutReached)); + if(timeoutReached) log.warn("Timeout reached. Check if {} is updated ",toUpdate.id()); + return querySEById(toUpdate.id()); + } + + + public static List queryById(String id) { + DiscoveryClient client = client(); + String queryString ="declare namespace ic = 'http://gcube-system.org/namespaces/informationsystem/registry'; "+ + "for $profiles in collection('/db/Profiles')//Document/Data/ic:Profile/Resource "+ + "where $profiles/ID/text() eq '"+id+"'"+ + " return $profiles"; + return client.submit(new QueryBox(queryString)); + } + + + public static ServiceEndpoint querySEById(String id) { + SimpleQuery query = queryFor(ServiceEndpoint.class); + + query.addCondition("$resource/ID/text() eq '"+id+"'"); + + DiscoveryClient client = clientFor(ServiceEndpoint.class); + + return client.submit(query).get(0); + } + + + public static String marshal(Resource res) { + ByteArrayOutputStream stream=new ByteArrayOutputStream(); + Resources.marshal(res, stream); + return stream.toString(); + } + + private AccessPoint getNewAccessPoint() { + AccessPoint toReturn=new AccessPoint(); + toReturn.credentials(encrypt(threddsAdminPassword), threddsAdminUser); + toReturn.description("Thredds Remote Management credentials"); + toReturn.name(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_REMOTE_MANAGEMENT_ACCESS)); + toReturn.address("https://"+getCurrentHostname()+"/thredds/admin/debug?catalogs/reinit"); + return toReturn; + } + + + static String decryptString(String toDecrypt){ + try{ + return StringEncrypter.getEncrypter().decrypt(toDecrypt); + }catch(Exception e) { + throw new RuntimeException("Unable to decrypt.",e); + } + } + + + static String encrypt(String toEncrypt) { + try{ + return StringEncrypter.getEncrypter().encrypt(toEncrypt); + }catch(Exception e) { + throw new RuntimeException("Unable to Encrypt.",e); + } + } + + static List queryForServiceEndpoints(String category, String platformName){ + log.debug("Querying for Service Endpoints [category : {} , platformName : {}]",category,platformName); + + SimpleQuery query = queryFor(ServiceEndpoint.class); + + query.addCondition("$resource/Profile/Category/text() eq '"+category+"'") + .addCondition("$resource/Profile/Platform/Name/text() eq '"+platformName+"'"); + + // .setResult("$resource/Profile/AccessPoint"); + + DiscoveryClient client = clientFor(ServiceEndpoint.class); + + return client.submit(query); + } + + static boolean isSameHost(String toTestHost,String toLookForHost) throws UnknownHostException { + log.debug("Checking same hosts {},{}",toTestHost,toLookForHost); + if(toTestHost.equalsIgnoreCase(toLookForHost)) return true; + else { + InetAddress[] toTestHostIPs=InetAddress.getAllByName(toTestHost); + InetAddress[] toLookForHostIPs=InetAddress.getAllByName(toLookForHost); + log.debug("Checking IPs. ToTestIPs {}, ToLookForIPs {} ",toTestHostIPs,toLookForHostIPs); + for(InetAddress toTestIP:toTestHostIPs) { + for(InetAddress toLookForIP:toLookForHostIPs) + if(toTestIP.equals(toLookForIP)) return true; + } + } + log.debug("HOSTS are different."); + return false; + } +} diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/TomcatSecurityHandler.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/TomcatSecurityHandler.java new file mode 100644 index 0000000..4dfaad1 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/TomcatSecurityHandler.java @@ -0,0 +1,29 @@ +package org.gcube.data.transfer.plugins.thredds; + +import java.io.File; +import java.io.IOException; + +import org.gcube.common.resources.gcore.utils.XPathHelper; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +public class TomcatSecurityHandler { + + XPathHelper helper; + + public TomcatSecurityHandler(String securityFile) throws SAXException, IOException { + helper=CommonXML.getHelper(CommonXML.getDocument(new File(securityFile))); + } + + public String getThreddsAdminUser() { + return getAdminUserElement().getAttribute("username"); + } + + public String getThreddsAdminPassword() { + return getAdminUserElement().getAttribute("password"); + } + + public Element getAdminUserElement() { + return (Element) helper.evaluateForNodes("//*[local-name()='user'][contains(@roles,'tdsConfig')]").item(0); + } +} diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/XMLCatalogHandler.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/XMLCatalogHandler.java new file mode 100644 index 0000000..c309684 --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/XMLCatalogHandler.java @@ -0,0 +1,193 @@ +package org.gcube.data.transfer.plugins.thredds; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; + +import javax.xml.transform.TransformerException; + +import org.gcube.common.resources.gcore.utils.XPathHelper; +import org.gcube.data.transfer.model.plugins.thredds.CatalogCollection; +import org.gcube.data.transfer.model.plugins.thredds.DataSetRoot; +import org.gcube.data.transfer.model.plugins.thredds.DataSetScan; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class XMLCatalogHandler { + + /* + * NB XML STRUCTURE IS AS FOLLOW + * catalog + * |-datasetRoot? + * |-datasetScan* + * |-dataset [dedicated_catalogs, only in main catalog file] + * |-catalogRef + * + */ + + + + private static final String DEDICATED_CATALOGS_DATASET_ID="VRE_Catalogs"; + + private static final String LINKED_CATALOGS_XPATH="//*[local-name()='catalogRef' and parent::node()[local-name()='dataset']]"; + private static final String DECLARED_DATASETSCANS="//*[local-name()='datasetScan']"; + private static final String DECLARED_DATASETROOT="//*[local-name()='datasetRoot']"; +// private static final String ROOT_CATALOG_XPATH="/[local-name()='catalog']"; + private static final String CATALOG_COLLECTION_XPATH="//*[local-name()='dataset'][child::node()[local-name()='catalogRef']]|//*[@ID='"+DEDICATED_CATALOGS_DATASET_ID+"']"; + + //PARAMETRIC + private static final String CATALOG_REFERENCE_BY_FILE="//*[local-name()='catalogRef'][@xlink:href='%s']"; + private static final String ELEMENT_BY_ID="//*[@ID='%s']"; + + + + + /* + * 1 - to expose reference + * 2 - path to catalog file + * 3 - ID + */ + private static final String catalogReferenceXMLPiece=""; + + + + + + + private XPathHelper helper; + private Document document; +// private Element documentNode; + private File currentCatalogFile; + + + public XMLCatalogHandler(File catalog) throws SAXException, IOException { + this.currentCatalogFile=catalog; + document=CommonXML.getDocument(catalog); + helper=CommonXML.getHelper(document); + } + + public void registerCatalog(File toRegisterCatalogFile,String toRegisterReference) throws SAXException, IOException { + String filename=toRegisterCatalogFile.getName(); + String ID=filename.contains(".")?filename.substring(0, filename.lastIndexOf('.')):filename; + + String toSetString=String.format(catalogReferenceXMLPiece, toRegisterReference,filename,ID); + log.info("Checking if file is already referenced.."); + + log.debug("Checking by filname {} ",filename); //Check filename presence + String referenceByFilenameXPATH=String.format(CATALOG_REFERENCE_BY_FILE,filename); + NodeList filenameReferencesNodelist=helper.evaluateForNodes(referenceByFilenameXPATH); + if(filenameReferencesNodelist.getLength()>0) { + log.info("Filename {} is already declared. Updateing reference..",filename); + CommonXML.addContent(referenceByFilenameXPATH,document,toSetString,helper,CommonXML.Position.replace); + }else { + + log.debug("Checking by ID {} ",ID); //else check by ID + String referenceByIDXPATH=String.format(ELEMENT_BY_ID, ID); + NodeList IDReferencesNodelist=helper.evaluateForNodes(referenceByIDXPATH); + if(IDReferencesNodelist.getLength()>0) { + log.info("ID {} found. Updateing reference..",ID); + CommonXML.addContent(referenceByIDXPATH,document,toSetString,helper,CommonXML.Position.replace); + }else { + + log.info("No similar entries found. Adding reference.."); //else add + String catalogCollectionXPATH=String.format(ELEMENT_BY_ID, DEDICATED_CATALOGS_DATASET_ID); + CommonXML.addContent(catalogCollectionXPATH,document,toSetString,helper,CommonXML.Position.last_child); + } + } + } + + + public void close() throws IOException, TransformerException { + CommonXML.writeOut(document, currentCatalogFile); + } + + + + + + /** + * Parses the current catalog file and linked ones in order to + * gather information on declared datasets + * + * + * @return + */ + public ThreddsCatalog getCatalogDescriptor() { + log.debug("loading catalogs from {} ",currentCatalogFile.getAbsolutePath()); + ThreddsCatalog toReturn=new ThreddsCatalog(); + toReturn.setCatalogFile(currentCatalogFile.getName()); + log.debug("Checking declared datasets in {} ",currentCatalogFile.getAbsolutePath()); + //get dataset root + NodeList datasetRootNodes=helper.evaluateForNodes(DECLARED_DATASETROOT); + if(datasetRootNodes.getLength()>0) { + Element rootNode=(Element) datasetRootNodes.item(0); + DataSetRoot root=new DataSetRoot(); + root.setLocation(rootNode.getAttribute("location")); + root.setPath(rootNode.getAttribute("path")); + toReturn.setDeclaredDataSetRoot(root); + } + //get dataset Scans + NodeList datasetScans=helper.evaluateForNodes(DECLARED_DATASETSCANS); + toReturn.setDeclaredDataSetScan(new HashSet()); + for(int i=0;i0) { + Element catalogCollectionNode=(Element) catalogCollectionNodes.item(0); + CatalogCollection collection=new CatalogCollection(); + + collection.setID(catalogCollectionNode.getAttribute("ID")); + collection.setName(catalogCollectionNode.getAttribute("name")); + log.debug("Found catalog collection ID {} , NAME {} ",collection.getID(),collection.getName()); + + collection.setLinkedCatalogs(new HashSet()); + NodeList linkedCatalogsNodelist=helper.evaluateForNodes(LINKED_CATALOGS_XPATH); + //load linked catalogs + for(int i=0;i params=invocation.getParameters(); + String catalogFile=params.get(RegisterCatalogPluginFactory.CATALOG_FILE); + if(catalogFile.contains(" ")) throw new PluginExecutionException("Invalid catalog filename "+catalogFile); + String catalogReference=params.get(RegisterCatalogPluginFactory.CATALOG_REFERENCE); + log.trace("Registering {} as {}",catalogFile,catalogReference); + + XMLCatalogHandler handler=instanceManager.mainCatalogHandler(); + handler.registerCatalog(new File(catalogFile),catalogReference); + handler.close(); + instanceManager.clearCache(); + return new ExecutionReport(invocation,"Registered catalog entry "+catalogReference,ExecutionReportFlag.SUCCESS); + }catch(Throwable t) { + log.error("Unable to register catalog.",t); + throw new PluginExecutionException("Unable to register catalog.",t); + } + } + +} diff --git a/src/main/java/org/gcube/data/transfer/plugins/thredds/catalog/RegisterCatalogPluginFactory.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/catalog/RegisterCatalogPluginFactory.java new file mode 100644 index 0000000..7ecaeaf --- /dev/null +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/catalog/RegisterCatalogPluginFactory.java @@ -0,0 +1,103 @@ +package org.gcube.data.transfer.plugins.thredds.catalog; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.data.transfer.plugin.AbstractPluginFactory; +import org.gcube.data.transfer.plugin.fails.ParameterException; +import org.gcube.data.transfer.plugin.fails.PluginExecutionException; +import org.gcube.data.transfer.plugin.fails.PluginInitializationException; +import org.gcube.data.transfer.plugin.fails.PluginShutDownException; +import org.gcube.data.transfer.plugin.model.DataTransferContext; +import org.gcube.data.transfer.plugins.thredds.ThreddsInstanceManager; +import org.gcube.data.transfer.plugins.thredds.XMLCatalogHandler; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RegisterCatalogPluginFactory extends AbstractPluginFactory{ + + static final String PLUGIN_ID="REGISTER_CATALOG"; + public static final String CATALOG_FILE="CATALOG_FILE"; + public static final String CATALOG_REFERENCE="CATALOG_REFERENCE"; + + + static final Map PARAMETERS_DESCRIPTION= new HashMap(); + + static{ + PARAMETERS_DESCRIPTION.put(CATALOG_REFERENCE, "[String value] The reference value under which the passed catalog should be registered."); + } + + @Override + public PluginInvocation checkInvocation(PluginInvocation arg0, String toRegisterCatalog) throws ParameterException { + log.trace("Checking invocation {} ",arg0); + Map params=arg0.getParameters(); + if(params==null||params.isEmpty()||(!params.containsKey(CATALOG_REFERENCE))||(params.get(CATALOG_REFERENCE)==null)) + throw new ParameterException(CATALOG_REFERENCE + "parameter is mandatory."); + arg0.getParameters().put(CATALOG_FILE, toRegisterCatalog); + log.debug("Checking if file {} is a valid catalog.."); + try { + new XMLCatalogHandler(new File(toRegisterCatalog)).getCatalogDescriptor(); + }catch(Throwable t) { + throw new ParameterException("Catalog File "+toRegisterCatalog+" is not valid.", t); + } + + + return arg0; + } + + @Override + public RegisterCatalogPlugin createWorker(PluginInvocation arg0) { + return new RegisterCatalogPlugin(arg0,ThreddsInstanceManager.get(ctx)); + } + + @Override + public String getDescription() { + return String.format("Registers a catalog file as <%s> under main thredds catalog.", CATALOG_REFERENCE); + } + + @Override + public String getID() { + return PLUGIN_ID; + } + + @Override + public Map getParameters() { + return PARAMETERS_DESCRIPTION; + } + + DataTransferContext ctx=null; + + @Override + public boolean init(DataTransferContext arg0) throws PluginInitializationException { + try{ + this.ctx=arg0; + ThreddsInstanceManager.get(ctx).updatePublishedInfo(); + return true; + }catch(Throwable t) { + throw new PluginInitializationException(t); + } + } + + @Override + public boolean shutDown() throws PluginShutDownException { + // TODO Auto-generated method stub + return false; + } + + @Override + public Object getInfo() throws PluginExecutionException{ + try{ + ThreddsInfo info=ThreddsInstanceManager.get(ctx).getInfo(); + log.info("Returning {} ",info); + return info; + }catch(Throwable t) { + log.error("Unable to gather catalog information",t); + throw new PluginExecutionException("Unable to gather information",t); + } + } + +} diff --git a/src/main/java/org/gcube/data/transfer/plugins/sis/SISPluginFactory.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java similarity index 72% rename from src/main/java/org/gcube/data/transfer/plugins/sis/SISPluginFactory.java rename to src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java index b5111cf..a88c283 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/sis/SISPluginFactory.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SISPluginFactory.java @@ -1,4 +1,4 @@ -package org.gcube.data.transfer.plugins.sis; +package org.gcube.data.transfer.plugins.thredds.sis; import java.io.File; import java.util.HashMap; @@ -10,52 +10,56 @@ import org.gcube.data.transfer.plugin.fails.ParameterException; import org.gcube.data.transfer.plugin.fails.PluginInitializationException; import org.gcube.data.transfer.plugin.fails.PluginShutDownException; import org.gcube.data.transfer.plugin.model.DataTransferContext; -import org.gcube.smartgears.configuration.application.ApplicationConfiguration; +import org.gcube.data.transfer.plugins.thredds.ThreddsInstanceManager; import lombok.extern.slf4j.Slf4j; @Slf4j public class SISPluginFactory extends AbstractPluginFactory { - - - + + + static final String PLUGIN_ID="SIS/GEOTK"; public static final String SOURCE_PARAMETER="SOURCE_FILE"; public static final String GEONETWORK_CATEGORY="GEONETWORK_CATEGORY"; public static final String GEONETWORK_STYLESHEET="GEONETWORK_STYLESHEET"; + public static final String VALIDATE_PARAMETER="VALIDATE"; + + static final Map PARAMETERS_DESCRIPTION= new HashMap(); static{ PARAMETERS_DESCRIPTION.put(GEONETWORK_CATEGORY, "[String value] GeoNetwork category for publiehd metadata. Default is 'Dataset'."); PARAMETERS_DESCRIPTION.put(GEONETWORK_STYLESHEET, "[String value] GeoNetwork stylesheet for publiehd metadata. Default is '_none_'."); + PARAMETERS_DESCRIPTION.put(VALIDATE_PARAMETER, "[Boolean value] Ask GeoNetwork to validate generated metadata. Default is 'true'."); } - + //INSTANCE public SISPluginFactory() { - + } - - String publicCatalogLocation=null; - - + + DataTransferContext ctx; + + @Override public PluginInvocation checkInvocation(PluginInvocation arg0,String transferredFile) throws ParameterException { log.debug("Setting default parameters for {} ",arg0); Map params=arg0.getParameters(); - + if(params==null||params.isEmpty()) params=new HashMap(); if(!params.containsKey(SOURCE_PARAMETER)) params.put(SOURCE_PARAMETER, transferredFile); if(!params.containsKey(GEONETWORK_CATEGORY)) params.put(GEONETWORK_CATEGORY, "Dataset"); if(!params.containsKey(GEONETWORK_STYLESHEET)) params.put(GEONETWORK_STYLESHEET, "_none_"); String source=params.get(SOURCE_PARAMETER); log.debug("Checking access to source {} ",source); - + if(source==null||source.length()==0) throw new ParameterException(SOURCE_PARAMETER+" cannot be null."); try{ File f=new File(source); @@ -66,15 +70,15 @@ public class SISPluginFactory extends AbstractPluginFactory { }catch(Exception e){ throw new ParameterException("Unable to access source file ",e); } - + arg0.setParameters(params); return arg0; - + } @Override public SisPlugin createWorker(PluginInvocation arg0) { - return new SisPlugin(arg0,ctx,publicCatalogLocation); + return new SisPlugin(arg0,ThreddsInstanceManager.get(ctx)); } @Override @@ -94,26 +98,13 @@ public class SISPluginFactory extends AbstractPluginFactory { @Override public boolean init(DataTransferContext ctx) throws PluginInitializationException { - log.trace("Loading configuration .. "); - this.ctx=ctx; - String threddsContentRoot=null; -// String threddsContentRoot=System.getenv("THREDDS_CONTENT_ROOT")+File.separator+"thredds"; -// log.info("Thredds catalog base path from ENV is {} ",threddsContentRoot); - - for(ApplicationConfiguration app:ctx.getCtx().container().configuration().apps()) { - log.debug("Found app {} ",app.context()); - if(app.context().equals("thredds")||app.context().equals("/thredds")) { - threddsContentRoot=app.persistence().location(); - log.info("Thredds catalog base path from Context is {} ",threddsContentRoot); - - // Found thredds - // Get catalog base folder -// app.persistence(). - } + try { + this.ctx=ctx; + ThreddsInstanceManager.get(ctx).updatePublishedInfo(); + return true; + }catch(Throwable t) { + throw new PluginInitializationException(t); } - if(threddsContentRoot==null) throw new PluginInitializationException("No Thredds instance found in context"); - publicCatalogLocation=threddsContentRoot+"/public/netcdf/"; - return true; } @Override @@ -121,7 +112,6 @@ public class SISPluginFactory extends AbstractPluginFactory { return true; } - - private DataTransferContext ctx; - + + } diff --git a/src/main/java/org/gcube/data/transfer/plugins/sis/SisPlugin.java b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SisPlugin.java similarity index 50% rename from src/main/java/org/gcube/data/transfer/plugins/sis/SisPlugin.java rename to src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SisPlugin.java index b4e41fc..332936d 100644 --- a/src/main/java/org/gcube/data/transfer/plugins/sis/SisPlugin.java +++ b/src/main/java/org/gcube/data/transfer/plugins/thredds/sis/SisPlugin.java @@ -1,4 +1,4 @@ -package org.gcube.data.transfer.plugins.sis; +package org.gcube.data.transfer.plugins.thredds.sis; import java.io.File; import java.util.Map; @@ -11,10 +11,12 @@ import org.apache.sis.xml.XML; import org.gcube.data.transfer.model.ExecutionReport; import org.gcube.data.transfer.model.ExecutionReport.ExecutionReportFlag; import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.model.plugins.thredds.DataSet; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog; import org.gcube.data.transfer.plugin.AbstractPlugin; import org.gcube.data.transfer.plugin.fails.PluginCleanupException; import org.gcube.data.transfer.plugin.fails.PluginExecutionException; -import org.gcube.data.transfer.plugin.model.DataTransferContext; +import org.gcube.data.transfer.plugins.thredds.ThreddsInstanceManager; import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions; import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocation; @@ -28,12 +30,12 @@ import lombok.extern.slf4j.Slf4j; public class SisPlugin extends AbstractPlugin { File tmp=null; - DataTransferContext ctx; - String publicCatalogLocation; - public SisPlugin(PluginInvocation invocation, DataTransferContext ctx, String publicCatalogLocation) { + + ThreddsInstanceManager instanceManager; + + public SisPlugin(PluginInvocation invocation, ThreddsInstanceManager instanceManager) { super(invocation); - this.ctx=ctx; - this.publicCatalogLocation=publicCatalogLocation; + this.instanceManager=instanceManager; } @Override @@ -53,6 +55,16 @@ public class SisPlugin extends AbstractPlugin { String dataStorePath=params.get(SISPluginFactory.SOURCE_PARAMETER); String category=params.get(SISPluginFactory.GEONETWORK_CATEGORY); String stylesheet=params.get(SISPluginFactory.GEONETWORK_STYLESHEET); + + Boolean validateMetadata=true; + + try { + if(params.containsKey(SISPluginFactory.VALIDATE_PARAMETER)) + validateMetadata=Boolean.parseBoolean(params.get(SISPluginFactory.VALIDATE_PARAMETER)); + }catch(Throwable t) { + log.warn("Unable to read validate parameter : "+params.get(SISPluginFactory.VALIDATE_PARAMETER),t); + } + File dataStore=new File(dataStorePath); log.debug("Extracting meta from {} ",dataStore.getAbsolutePath()); Metadata meta=getMetaFromFile(dataStore); @@ -63,39 +75,32 @@ public class SisPlugin extends AbstractPlugin { org.gcube.spatial.data.sdi.interfaces.Metadata metadataInterface=SDIAbstractPlugin.metadata().build(); - String hostname=ctx.getCtx().container().configuration().hostname(); - String filename=dataStore.getName(); - String catalog=getCatalogFromPath(dataStorePath); + String hostname=instanceManager.getCurrentHostname(); + String filename=dataStore.getName(); + ThreddsCatalog catalog=instanceManager.getInfo().getCatalogByFittingLocation(dataStorePath); + log.debug("Catalog for transferred File at {} is {} ",dataStorePath,catalog); + DataSet catalogDataset=catalog.getDataSetFromLocation(dataStorePath); + String datasetSubPath=dataStorePath.substring(catalogDataset.getLocation().length(), dataStorePath.lastIndexOf("/")); + String datasetPath=catalogDataset.getPath()+datasetSubPath; + + + log.info("Publishing dataset {} with path {} ",dataStorePath,datasetPath); Set invocations=new TemplateInvocationBuilder(). - threddsOnlineResources(hostname,filename, catalog).get(); + threddsOnlineResources(hostname,filename, datasetPath).get(); MetadataPublishOptions options=new MetadataPublishOptions(invocations); options.setGeonetworkCategory(category); options.setGeonetworkStyleSheet(stylesheet); - options.setValidate(false); + options.setMakePublic(catalogDataset.getPath().startsWith("public")); + + options.setValidate(validateMetadata); MetadataReport report=metadataInterface.pushMetadata(tmp, options); log.debug("Metadata report is {} ",report); return new ExecutionReport(invocation, "Published/Updated meta with id : "+report.getPublishedID()+" , UUID "+report.getPublishedUUID(),ExecutionReportFlag.SUCCESS); - -//// TemplateManager.getTHREDDSLinks(new ThreddsLinkRequest(hostname, filename, catalog, gisViewerLink)) -// // TODO Info from infrastructure -// try{ -// long id=publishMetadata(tmp, category, stylesheet,getGNPublisher()); -// return new ExecutionReport(invocation, "Published meta with id : "+id, ExecutionReportFlag.SUCCESS); -// }catch(GNServerException e){ -// //Fail can be due to already existing uuid. -// log.debug("Trying to update existing uuid"); -// -// MetadataHandler metaHandler=new MetadataHandler(tmp); -// String uuid=metaHandler.getUUID(); -// long id= updateMetadata(tmp, uuid, category, stylesheet,getGNPublisher()); -// return new ExecutionReport(invocation, "Updated meta with id : "+id+", UUID : "+uuid, ExecutionReportFlag.SUCCESS); -// } - }catch(DataStoreException e){ @@ -106,55 +111,11 @@ public class SisPlugin extends AbstractPlugin { throw new PluginExecutionException("Unexpected error while generating meta.",t); } } - -// public GeoNetworkPublisher getGNPublisher() throws Exception{ -// if(publisher==null){ -// publisher=GeoNetwork.get(); -// publisher.login(LoginLevel.DEFAULT); -// } -// return publisher; -// } -// public static final Metadata getMetaFromFile(Object dataStore) throws UnsupportedStorageException, DataStoreException{ return DataStores.open(dataStore).getMetadata(); } - private String getCatalogFromPath(String path){ - return getCatalogFromPath(path, publicCatalogLocation); - } - public static String getCatalogFromPath(String path, String publicCatalogLocation) { - log.debug("Getting catalog from path {} ",path); - String catalog=path.substring(publicCatalogLocation.length()); - if(catalog.contains("/")) { - catalog=catalog.substring(0, catalog.lastIndexOf("/")); - log.debug("Subcatalog found {} ",catalog); - }else { - log.debug("No catalog found"); - catalog=null; - } - return catalog; - } -// -// public long publishMetadata(File toPublish,String category, String stylesheet, GeoNetworkPublisher publisher) throws Exception{ -// GNInsertConfiguration config=publisher.getCurrentUserConfiguration(category, stylesheet); -// config.setValidate(false); -// long toReturn= publisher.insertMetadata(config, toPublish); -// return toReturn; -// } -// -// public long updateMetadata(File toPublish,String uuid,String category,String styleSheet, GeoNetworkPublisher publisher)throws Exception{ -// long id=0l; -// if(publisher.getConfiguration().getGeoNetworkVersion().equals(Version.DUE)){ -// GNSearchRequest req=new GNSearchRequest(); -// req.addParam(GNSearchRequest.Param.any,uuid); -// -// GNSearchResponse resp=publisher.query(req); -// id=resp.iterator().next().getId(); -// }else id=publisher.getInfo(uuid).getId(); -// publisher.updateMetadata(id, toPublish); -// return id; -// } } diff --git a/src/main/resources/META-INF/services/org.gcube.data.transfer.plugin.AbstractPluginFactory b/src/main/resources/META-INF/services/org.gcube.data.transfer.plugin.AbstractPluginFactory index 1b0e759..a785158 100644 --- a/src/main/resources/META-INF/services/org.gcube.data.transfer.plugin.AbstractPluginFactory +++ b/src/main/resources/META-INF/services/org.gcube.data.transfer.plugin.AbstractPluginFactory @@ -1 +1,2 @@ -org.gcube.data.transfer.plugins.sis.SISPluginFactory \ No newline at end of file +org.gcube.data.transfer.plugins.thredds.sis.SISPluginFactory +org.gcube.data.transfer.plugins.thredds.catalog.RegisterCatalogPluginFactory \ No newline at end of file diff --git a/src/main/resources/thredds.properties b/src/main/resources/thredds.properties new file mode 100644 index 0000000..c47cd2a --- /dev/null +++ b/src/main/resources/thredds.properties @@ -0,0 +1,9 @@ +th.se.category=Gis +th.se.platform=Thredds +th.se.remoteManagement.access=Remote_Management + +#IS +is.registration.timeout=60000 + +#Context +context.loading.timeout=600000 \ No newline at end of file diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/TestAddCatalog.java b/src/test/java/org/gcube/data/transfer/plugins/sis/TestAddCatalog.java new file mode 100644 index 0000000..ad897de --- /dev/null +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/TestAddCatalog.java @@ -0,0 +1,21 @@ +package org.gcube.data.transfer.plugins.sis; + +import java.io.File; +import java.io.IOException; + +import javax.xml.transform.TransformerException; + +import org.gcube.data.transfer.plugins.thredds.XMLCatalogHandler; +import org.xml.sax.SAXException; + +public class TestAddCatalog { + + public static void main(String[] args) throws SAXException, IOException, TransformerException { + String mainCatalogPath="/home/fabio/Desktop/catalog.xml"; + XMLCatalogHandler handler=new XMLCatalogHandler(new File(mainCatalogPath)); + System.out.println("Catalogs : "+handler.getCatalogDescriptor()); + handler.registerCatalog(new File("/home/fabio/Desktop/Tuna_Atlas_VRE_catalog.xml"), "testReplace2"); + handler.close(); + } + +} diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetCatalog.java b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetCatalog.java index cba3a3c..ddaeb64 100644 --- a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetCatalog.java +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetCatalog.java @@ -1,11 +1,32 @@ package org.gcube.data.transfer.plugins.sis; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.data.transfer.library.client.AuthorizationFilter; +import org.gcube.data.transfer.model.plugins.thredds.DataSet; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.glassfish.jersey.client.ClientConfig; + public class TestGetCatalog { public static void main(String[] args) { - String path="/home/gcube/tomcat/content/thredds/public/netcdf/myCatalog/CERSAT-GLO-CLIM_WIND_L4-OBS_FULL_TIME_SERIE_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366217956317.nc"; - String publicLocation="/home/gcube/tomcat/content/thredds/public/netcdf/"; - System.out.println(SisPlugin.getCatalogFromPath(path, publicLocation)); + SecurityTokenProvider.instance.set("f851ba11-bd3e-417a-b2c2-753b02bac506-98187548"); + String dataStorePath="/data/content/thredds/devVRE/indian_ocean_catch_5deg_1m_1952_11_01_2016_01_01_tunaatlasIRD_level1.nc"; + //getting threddsInfo from thredds-d + Client client=ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class)); + WebTarget target=client.target(String.format("https://%s/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG", "thredds-d-d4s.d4science.org")); + ThreddsInfo info=target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class); + System.out.println(info.getCatalogByFittingLocation(dataStorePath)); + DataSet catalogDataset=info.getDataSetFromLocation(dataStorePath); + + + String datasetSubPath=dataStorePath.substring(catalogDataset.getLocation().length(), dataStorePath.lastIndexOf("/")); + String datasetPath=catalogDataset.getPath()+datasetSubPath; + System.out.println("Path : "+datasetPath); } } diff --git a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java index 6b7304d..7f2980e 100644 --- a/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java +++ b/src/test/java/org/gcube/data/transfer/plugins/sis/TestGetMetadata.java @@ -16,6 +16,7 @@ import java.security.NoSuchAlgorithmException; import org.apache.sis.storage.DataStoreException; import org.apache.sis.storage.UnsupportedStorageException; import org.gcube.data.transfer.model.TransferTicket.Status; +import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin; public class TestGetMetadata {