Merged
git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/sis-geotk-plugin@181739 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
51de029967
commit
4b04fd7224
|
|
@ -1,5 +1,11 @@
|
|||
<ReleaseNotes>
|
||||
<Changeset component="sis-geotk-plugin-1.0.0" date="2017-05-22">
|
||||
<Change>First Release</Change>
|
||||
</Changeset>
|
||||
<Changeset component="sis-geotk-plugin-1.0.1" date="2017-10-12">
|
||||
<Change>Bug fix</Change>
|
||||
</Changeset>
|
||||
<Changeset component="sis-geotk-plugin-1.1.0" date="2017-12-02">
|
||||
<Change>RegisterCatalogPlugin</Change>
|
||||
</Changeset>
|
||||
</ReleaseNotes>
|
||||
20
pom.xml
20
pom.xml
|
|
@ -4,11 +4,11 @@
|
|||
<parent>
|
||||
<groupId>org.gcube.tools</groupId>
|
||||
<artifactId>maven-parent</artifactId>
|
||||
<version>LATEST</version>
|
||||
<version>1.1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<groupId>org.gcube.data.transfer</groupId>
|
||||
<artifactId>sis-geotk-plugin</artifactId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>1.1.1-SNAPSHOT</version>
|
||||
<name>Sis/GeoToolkit plugin</name>
|
||||
<description>Apache Sis/Geotk plugin for data-transfer-service</description>
|
||||
|
||||
|
|
@ -52,6 +52,15 @@
|
|||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.resources</groupId>
|
||||
<artifactId>registry-publisher</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.core</groupId>
|
||||
<artifactId>common-encryption</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency> -->
|
||||
<!-- <groupId>org.gcube.spatial.data</groupId> -->
|
||||
<!-- <artifactId>geonetwork</artifactId> -->
|
||||
|
|
@ -101,7 +110,12 @@
|
|||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data.transfer</groupId>
|
||||
<artifactId>data-transfer-library</artifactId>
|
||||
<version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
package org.gcube.data.transfer.plugins.thredds;
|
||||
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import org.gcube.data.transfer.plugin.model.DataTransferContext;
|
||||
import org.gcube.smartgears.configuration.application.ApplicationConfiguration;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class ApplicationConfigurationRetriever implements Callable<ApplicationConfiguration>{
|
||||
|
||||
private DataTransferContext ctx;
|
||||
|
||||
private long timeout=LocalConfiguration.getTTL(LocalConfiguration.CONTEXT_LOADING_TIMETOUT);
|
||||
|
||||
public ApplicationConfigurationRetriever(DataTransferContext ctx) {
|
||||
this.ctx=ctx;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public ApplicationConfiguration call() throws Exception {
|
||||
ApplicationConfiguration toReturn=null;
|
||||
log.info("Waiting for thredds application to be loaded");
|
||||
long startTime=System.currentTimeMillis();
|
||||
while(toReturn==null&(System.currentTimeMillis()-startTime<timeout)) {
|
||||
try{Thread.sleep(1000);
|
||||
}catch(InterruptedException e) {}
|
||||
for(ApplicationConfiguration app:ctx.getCtx().container().configuration().apps()) {
|
||||
log.debug("Found app {} ",app.context());
|
||||
if(app.context().equals("thredds")||app.context().equals("/thredds")) {
|
||||
toReturn=app;
|
||||
}
|
||||
}
|
||||
}
|
||||
log.info("Retrieved {} after {}ms ",toReturn,(System.currentTimeMillis()-startTime));
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package org.gcube.data.transfer.plugins.thredds;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@AllArgsConstructor
|
||||
public class CatalogDescriptor {
|
||||
|
||||
private String ID;
|
||||
private String name;
|
||||
private String title;
|
||||
private String catalogFile;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
package org.gcube.data.transfer.plugins.thredds;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.TransformerFactory;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.stream.StreamResult;
|
||||
|
||||
import org.gcube.common.resources.gcore.utils.XPathHelper;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
public class CommonXML {
|
||||
|
||||
|
||||
|
||||
static Transformer transformer =null;
|
||||
static DocumentBuilder docBuilder =null;
|
||||
|
||||
|
||||
static HashMap<String,String> namespaces=new HashMap<String,String>();
|
||||
|
||||
|
||||
|
||||
|
||||
static{
|
||||
try{
|
||||
DocumentBuilderFactory factory=DocumentBuilderFactory.newInstance();
|
||||
factory.setNamespaceAware(true);
|
||||
docBuilder = factory.newDocumentBuilder();
|
||||
|
||||
|
||||
TransformerFactory transformerFactory = TransformerFactory.newInstance();
|
||||
transformer = transformerFactory.newTransformer();
|
||||
|
||||
|
||||
namespaces.put("xmlns", "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0");
|
||||
namespaces.put("xlink", "http://www.w3.org/1999/xlink");
|
||||
namespaces.put("xsi", "http://www.w3.org/2001/XMLSchema-instance");
|
||||
// namespaces.put("gml", "http://www.opengis.net/gml");
|
||||
// namespaces.put("xsi", "http://www.w3.org/2001/XMLSchema-instance");
|
||||
// namespaces.put("gmi", "http://www.isotc211.org/2005/gmi");
|
||||
// namespaces.put("gmx", "http://www.isotc211.org/2005/gmx");
|
||||
|
||||
|
||||
|
||||
|
||||
}catch(Exception e){
|
||||
throw new RuntimeException("Unable to init Fixer ",e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Document getDocument(File xmlFile) throws SAXException, IOException {
|
||||
log.debug("Parsing {} ",xmlFile.getAbsolutePath());
|
||||
InputSource inputSource = new InputSource(new FileInputStream(xmlFile));
|
||||
//Get document owner
|
||||
Element documentNode = docBuilder.parse(inputSource).getDocumentElement();
|
||||
return documentNode.getOwnerDocument();
|
||||
}
|
||||
|
||||
public static XPathHelper getHelper(Node root){
|
||||
XPathHelper toReturn =new XPathHelper(root);
|
||||
for(Entry<String,String> entry:namespaces.entrySet())
|
||||
toReturn.addNamespace(entry.getKey(), entry.getValue());
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
public static void writeOut(Document document, File destination) throws IOException, TransformerException {
|
||||
DOMSource source = new DOMSource(document);
|
||||
StreamResult result = new StreamResult(new FileWriter(destination));
|
||||
transformer.transform(source, result);
|
||||
result.getWriter().flush();
|
||||
result.getWriter().close();
|
||||
}
|
||||
|
||||
|
||||
public static enum Position{
|
||||
sibling_after,sibling_before,first_child,last_child,replace
|
||||
}
|
||||
|
||||
public static void addContent(String path, Document doc, String toAddContent, XPathHelper documentHelper,Position position) throws SAXException, IOException{
|
||||
NodeList nodelist=documentHelper.evaluateForNodes(path);
|
||||
if(nodelist==null||nodelist.getLength()==0) throw new RuntimeException("Path "+path+" not found in document");
|
||||
// if(nodelist.getLength()>1) throw new RuntimeException("Invalid Path "+path+"."+nodelist.getLength()+" entries found");
|
||||
Node targetNode=nodelist.item(0);
|
||||
|
||||
Document online=docBuilder.parse(new ByteArrayInputStream(toAddContent.getBytes()));
|
||||
Node toAdd=doc.importNode(online.getDocumentElement(), true);
|
||||
switch(position){
|
||||
case first_child: {
|
||||
targetNode.insertBefore(toAdd, targetNode.getFirstChild());
|
||||
break;
|
||||
}
|
||||
case last_child:{targetNode.appendChild(toAdd);
|
||||
break;}
|
||||
case replace : {
|
||||
Node parent=targetNode.getParentNode();
|
||||
parent.replaceChild(toAdd, targetNode);
|
||||
break;
|
||||
}
|
||||
case sibling_after :{
|
||||
Node currentlyNext=targetNode.getNextSibling();
|
||||
Node parent=targetNode.getParentNode();
|
||||
if(currentlyNext!=null)parent.insertBefore(toAdd, currentlyNext);
|
||||
else parent.appendChild(toAdd);
|
||||
break;
|
||||
}
|
||||
case sibling_before :{
|
||||
Node parent=targetNode.getParentNode();
|
||||
parent.insertBefore(toAdd, targetNode);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package org.gcube.data.transfer.plugins.thredds;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
public class LocalConfiguration {
|
||||
|
||||
final static public String THREDDS_SE_CATEGORY="th.se.category";
|
||||
final static public String THREDDS_SE_PLATFORM="th.se.platform";
|
||||
final static public String THREDDS_SE_REMOTE_MANAGEMENT_ACCESS="th.se.remoteManagement.access";
|
||||
|
||||
final static public String IS_REGISTRATION_TIMEOUT="is.registration.timeout";
|
||||
|
||||
final static public String CONTEXT_LOADING_TIMETOUT="context.loading.timeout";
|
||||
|
||||
static LocalConfiguration instance=null;
|
||||
|
||||
|
||||
public static synchronized LocalConfiguration get(){
|
||||
if(instance==null)
|
||||
instance=new LocalConfiguration();
|
||||
return instance;
|
||||
}
|
||||
|
||||
private Properties props=new Properties();
|
||||
|
||||
private LocalConfiguration() {
|
||||
try{
|
||||
props.load(this.getClass().getResource("/thredds.properties").openStream());
|
||||
}catch(Exception e){
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static String getProperty(String property){
|
||||
return get().props.getProperty(property);
|
||||
}
|
||||
|
||||
public static Long getTTL(String property) {
|
||||
return Long.parseLong(getProperty(property));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,394 @@
|
|||
package org.gcube.data.transfer.plugins.thredds;
|
||||
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.client;
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ConcurrentSkipListSet;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.encryption.StringEncrypter;
|
||||
import org.gcube.common.resources.gcore.Resource;
|
||||
import org.gcube.common.resources.gcore.Resources;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint.Profile;
|
||||
import org.gcube.common.resources.gcore.common.Platform;
|
||||
import org.gcube.common.resources.gcore.utils.Group;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
|
||||
import org.gcube.data.transfer.plugin.model.DataTransferContext;
|
||||
import org.gcube.informationsystem.publisher.RegistryPublisher;
|
||||
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
|
||||
import org.gcube.smartgears.configuration.application.ApplicationConfiguration;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import lombok.Synchronized;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
@Slf4j
|
||||
public class ThreddsInstanceManager {
|
||||
|
||||
protected static ThreddsInstanceManager instance=null;
|
||||
|
||||
protected static final ConcurrentSkipListSet<String> checkedTokens=new ConcurrentSkipListSet<>();
|
||||
|
||||
|
||||
@Synchronized
|
||||
public static ThreddsInstanceManager get(DataTransferContext ctx) {
|
||||
if(instance==null)
|
||||
instance= new ThreddsInstanceManager(ctx);
|
||||
return instance;
|
||||
}
|
||||
|
||||
// ***************** INSTANCE LOGIC
|
||||
|
||||
|
||||
protected ThreddsInfo cachedInfo=null;
|
||||
protected ExecutorService executor=Executors.newSingleThreadExecutor();
|
||||
//loaded at construction time
|
||||
protected DataTransferContext ctx=null;
|
||||
protected ApplicationConfiguration threddsConfig=null;
|
||||
protected String threddsAdminUser;
|
||||
protected String threddsAdminPassword;
|
||||
protected String threddsPersistenceLocation;
|
||||
protected String threddsVersionString;
|
||||
protected String currentHostname;
|
||||
protected String currentGHNId;
|
||||
|
||||
protected ThreddsInstanceManager(DataTransferContext context) {
|
||||
log.warn("Instance Creation. Should happen only once. Loading information from context..");
|
||||
this.ctx=context;
|
||||
currentHostname=ctx.getCtx().container().configuration().hostname();
|
||||
currentGHNId=ctx.getCtx().container().id();
|
||||
String tomcatSecurityPath=System.getenv("WEB_CONTAINER_HOME")+"/conf/tomcat-users.xml";
|
||||
|
||||
log.info("Loading security from {} ",tomcatSecurityPath);
|
||||
try{
|
||||
TomcatSecurityHandler tomcatHandler=new TomcatSecurityHandler(tomcatSecurityPath);
|
||||
threddsAdminUser=tomcatHandler.getThreddsAdminUser();
|
||||
threddsAdminPassword=tomcatHandler.getThreddsAdminPassword();
|
||||
}catch(Exception e) {
|
||||
throw new RuntimeException("Unable to parse security file "+tomcatSecurityPath,e);
|
||||
}
|
||||
log.info("Looking for Thredds Application Configuration.. ");
|
||||
|
||||
|
||||
//Use Future
|
||||
Future<ApplicationConfiguration> future=executor.submit(new ApplicationConfigurationRetriever(ctx));
|
||||
try {
|
||||
threddsConfig=future.get();
|
||||
if(threddsConfig==null) throw new Exception("Returned Application Configuration is null");
|
||||
threddsPersistenceLocation=threddsConfig.persistence().location();
|
||||
threddsVersionString=threddsConfig.version();
|
||||
}catch(Exception e) {
|
||||
throw new RuntimeException("Unable to find Application Configuration for thredds.",e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
* @return
|
||||
* @throws IOException
|
||||
* @throws SAXException
|
||||
* @throws Exception
|
||||
*/
|
||||
public synchronized ThreddsInfo getInfo() throws SAXException, IOException {
|
||||
if(cachedInfo==null) {
|
||||
log.info("Loading ThreddsInfo..");
|
||||
|
||||
String threddsContentRoot=getContentRoot();
|
||||
|
||||
|
||||
log.debug("Found content root at {} ",threddsContentRoot);
|
||||
|
||||
//Host
|
||||
ThreddsInfo info=new ThreddsInfo();
|
||||
info.setHostname(currentHostname);
|
||||
info.setGhnId(currentGHNId);
|
||||
info.setLocalBasePath(threddsPersistenceLocation);
|
||||
info.setInstanceBaseUrl("http://"+info.getHostname()+"/thredds");
|
||||
|
||||
|
||||
String mainCatalogPath=threddsContentRoot+"/catalog.xml";
|
||||
|
||||
log.info("Loading catalog information from {} ",mainCatalogPath);
|
||||
|
||||
XMLCatalogHandler handler=new XMLCatalogHandler(new File(mainCatalogPath));
|
||||
info.setCatalog(handler.getCatalogDescriptor());
|
||||
|
||||
|
||||
//tomcat security
|
||||
info.setAdminPassword(threddsAdminPassword);
|
||||
info.setAdminUser(threddsAdminUser);
|
||||
|
||||
//version
|
||||
String[] splittedVersion=threddsVersionString.split("\\.");
|
||||
info.setVersion(Integer.parseInt(splittedVersion[0]));
|
||||
info.setMinor(Integer.parseInt(splittedVersion[1]));
|
||||
info.setRevision(Integer.parseInt(splittedVersion[2]));
|
||||
log.info("Loaded ThreddsInfo is {} ",info);
|
||||
cachedInfo=info;
|
||||
}
|
||||
return cachedInfo;
|
||||
}
|
||||
|
||||
public synchronized void clearCache() {
|
||||
log.debug("Clearing cache..");
|
||||
cachedInfo=null;
|
||||
}
|
||||
|
||||
public String getCurrentHostname() {
|
||||
return currentHostname;
|
||||
}
|
||||
|
||||
public String getMainCatalogFile(){
|
||||
return getContentRoot()+"/catalog.xml";
|
||||
}
|
||||
|
||||
public String getContentRoot() {
|
||||
return threddsPersistenceLocation;
|
||||
}
|
||||
|
||||
|
||||
public XMLCatalogHandler mainCatalogHandler() throws SAXException, IOException, Exception {
|
||||
return new XMLCatalogHandler(new File(getMainCatalogFile()));
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
public void updatePublishedInfo() throws Exception {
|
||||
String token=SecurityTokenProvider.instance.get();
|
||||
log.info("Checking IS with token {} ",token);
|
||||
if(!checkedTokens.contains(token)) {
|
||||
checkedTokens.add(token);
|
||||
getInfo();
|
||||
String currentHostname=cachedInfo.getHostname();
|
||||
|
||||
log.info("Checking IS Information, host is {}",currentHostname);
|
||||
|
||||
List<ServiceEndpoint> currentEndpoints=queryForServiceEndpoints(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_CATEGORY),
|
||||
LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_PLATFORM));
|
||||
|
||||
ServiceEndpoint toCheck=null;
|
||||
|
||||
//Checking by host
|
||||
log.debug("Found {} Service Endpoints, checking by hostname {} ",currentEndpoints.size());
|
||||
for(ServiceEndpoint se:currentEndpoints) {
|
||||
String host=se.profile().runtime().hostedOn();
|
||||
try{
|
||||
if(isSameHost(host, currentHostname)) {toCheck=se;
|
||||
break;}
|
||||
}catch(Throwable t) {
|
||||
log.warn("Unable to check Host {} ",host,t);
|
||||
}
|
||||
}
|
||||
|
||||
if(toCheck==null) {
|
||||
log.info("ServiceEndpoint not found, going to create one..");
|
||||
// CREATE NEW
|
||||
ServiceEndpoint newSE=getNewServiceEndpoint();
|
||||
updateAndWait(newSE, true);
|
||||
|
||||
}else {
|
||||
// Check found
|
||||
boolean updateSE=true;
|
||||
String adminAPName=LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_REMOTE_MANAGEMENT_ACCESS);
|
||||
log.debug("Looking for Access Point {} ",adminAPName);
|
||||
Group<AccessPoint> existentAP=toCheck.profile().accessPoints();
|
||||
|
||||
|
||||
boolean addAccessPoint=true;
|
||||
for(AccessPoint ap:existentAP)
|
||||
if(ap.name().equals(adminAPName)) {
|
||||
addAccessPoint=false;
|
||||
// FOUND AP
|
||||
String pwd=decryptString(ap.password());
|
||||
if(ap.username().equalsIgnoreCase(threddsAdminUser)&&pwd.equalsIgnoreCase(threddsAdminPassword)) {
|
||||
log.info("ServiceEndopint is up to date.");
|
||||
updateSE=false;
|
||||
}else {
|
||||
// AP is not up to date
|
||||
ap.credentials(threddsAdminPassword, threddsAdminUser);
|
||||
}
|
||||
}
|
||||
|
||||
if(updateSE) {
|
||||
log.debug("Need to update SE... ");
|
||||
if(addAccessPoint) {
|
||||
log.debug("Access point {} not found. Adding it.. ",adminAPName);
|
||||
existentAP.add(getNewAccessPoint());
|
||||
}
|
||||
ServiceEndpoint updated=updateAndWait(toCheck,false);
|
||||
log.info("Updated {} ",updated);
|
||||
}
|
||||
|
||||
}
|
||||
}else log.info("Skipping token {}, already checked.",token);
|
||||
|
||||
}
|
||||
|
||||
private static String registerServiceEndpoint(ServiceEndpoint toRegister) {
|
||||
RegistryPublisher rp=RegistryPublisherFactory.create();
|
||||
Resource r=rp.create(toRegister);
|
||||
return r.id();
|
||||
}
|
||||
|
||||
public static ServiceEndpoint update(ServiceEndpoint toUpdate) {
|
||||
RegistryPublisher rp=RegistryPublisherFactory.create();
|
||||
return rp.update(toUpdate);
|
||||
}
|
||||
|
||||
private ServiceEndpoint getNewServiceEndpoint() {
|
||||
ServiceEndpoint toReturn=new ServiceEndpoint();
|
||||
|
||||
Profile profile=toReturn.newProfile();
|
||||
profile.category(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_CATEGORY));
|
||||
profile.name("Thredds on "+cachedInfo.getHostname());
|
||||
profile.description("Thredds on "+cachedInfo.getHostname());
|
||||
|
||||
// TODO Gather info on version
|
||||
Platform platform=profile.newPlatform();
|
||||
platform.version((short)cachedInfo.getVersion());
|
||||
platform.minorVersion((short)cachedInfo.getMinor());
|
||||
platform.revisionVersion((short)cachedInfo.getRevision());
|
||||
platform.buildVersion((short)cachedInfo.getBuild());
|
||||
platform.name(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_PLATFORM));
|
||||
|
||||
org.gcube.common.resources.gcore.ServiceEndpoint.Runtime runtime=profile.newRuntime();
|
||||
runtime.ghnId(cachedInfo.getGhnId());
|
||||
runtime.hostedOn(cachedInfo.getHostname());
|
||||
runtime.status("READY");
|
||||
|
||||
profile.accessPoints().add(getNewAccessPoint());
|
||||
return toReturn;
|
||||
|
||||
}
|
||||
|
||||
private static ServiceEndpoint updateAndWait(ServiceEndpoint toUpdate,boolean isNew) {
|
||||
boolean equals=true;
|
||||
boolean timeoutReached=false;
|
||||
long timeout=LocalConfiguration.getTTL(LocalConfiguration.IS_REGISTRATION_TIMEOUT);
|
||||
log.info("Going to register {}. Timeout is {} ",toUpdate.id(),timeout);
|
||||
String toUpdateString=marshal(toUpdate);
|
||||
log.debug("Serialized resource is {} ",toUpdateString);
|
||||
if(isNew) registerServiceEndpoint(toUpdate);
|
||||
else update(toUpdate);
|
||||
long updateTime=System.currentTimeMillis();
|
||||
String updatedString=null;
|
||||
do {
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
} catch (InterruptedException e) {}
|
||||
List<String> byIdResults=queryById(toUpdate.id());
|
||||
if(byIdResults.isEmpty()) {
|
||||
equals=false;
|
||||
}else {
|
||||
updatedString=byIdResults.get(0);
|
||||
equals=toUpdateString.equals(updatedString);
|
||||
}
|
||||
timeoutReached=(System.currentTimeMillis()-updateTime)>timeout;
|
||||
}while(equals&&(!timeoutReached));
|
||||
if(timeoutReached) log.warn("Timeout reached. Check if {} is updated ",toUpdate.id());
|
||||
return querySEById(toUpdate.id());
|
||||
}
|
||||
|
||||
|
||||
public static List<String> queryById(String id) {
|
||||
DiscoveryClient<String> client = client();
|
||||
String queryString ="declare namespace ic = 'http://gcube-system.org/namespaces/informationsystem/registry'; "+
|
||||
"for $profiles in collection('/db/Profiles')//Document/Data/ic:Profile/Resource "+
|
||||
"where $profiles/ID/text() eq '"+id+"'"+
|
||||
" return $profiles";
|
||||
return client.submit(new QueryBox(queryString));
|
||||
}
|
||||
|
||||
|
||||
public static ServiceEndpoint querySEById(String id) {
|
||||
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||
|
||||
query.addCondition("$resource/ID/text() eq '"+id+"'");
|
||||
|
||||
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||
|
||||
return client.submit(query).get(0);
|
||||
}
|
||||
|
||||
|
||||
public static String marshal(Resource res) {
|
||||
ByteArrayOutputStream stream=new ByteArrayOutputStream();
|
||||
Resources.marshal(res, stream);
|
||||
return stream.toString();
|
||||
}
|
||||
|
||||
private AccessPoint getNewAccessPoint() {
|
||||
AccessPoint toReturn=new AccessPoint();
|
||||
toReturn.credentials(encrypt(threddsAdminPassword), threddsAdminUser);
|
||||
toReturn.description("Thredds Remote Management credentials");
|
||||
toReturn.name(LocalConfiguration.getProperty(LocalConfiguration.THREDDS_SE_REMOTE_MANAGEMENT_ACCESS));
|
||||
toReturn.address("https://"+getCurrentHostname()+"/thredds/admin/debug?catalogs/reinit");
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
static String decryptString(String toDecrypt){
|
||||
try{
|
||||
return StringEncrypter.getEncrypter().decrypt(toDecrypt);
|
||||
}catch(Exception e) {
|
||||
throw new RuntimeException("Unable to decrypt.",e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static String encrypt(String toEncrypt) {
|
||||
try{
|
||||
return StringEncrypter.getEncrypter().encrypt(toEncrypt);
|
||||
}catch(Exception e) {
|
||||
throw new RuntimeException("Unable to Encrypt.",e);
|
||||
}
|
||||
}
|
||||
|
||||
static List<ServiceEndpoint> queryForServiceEndpoints(String category, String platformName){
|
||||
log.debug("Querying for Service Endpoints [category : {} , platformName : {}]",category,platformName);
|
||||
|
||||
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||
|
||||
query.addCondition("$resource/Profile/Category/text() eq '"+category+"'")
|
||||
.addCondition("$resource/Profile/Platform/Name/text() eq '"+platformName+"'");
|
||||
|
||||
// .setResult("$resource/Profile/AccessPoint");
|
||||
|
||||
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||
|
||||
return client.submit(query);
|
||||
}
|
||||
|
||||
static boolean isSameHost(String toTestHost,String toLookForHost) throws UnknownHostException {
|
||||
log.debug("Checking same hosts {},{}",toTestHost,toLookForHost);
|
||||
if(toTestHost.equalsIgnoreCase(toLookForHost)) return true;
|
||||
else {
|
||||
InetAddress[] toTestHostIPs=InetAddress.getAllByName(toTestHost);
|
||||
InetAddress[] toLookForHostIPs=InetAddress.getAllByName(toLookForHost);
|
||||
log.debug("Checking IPs. ToTestIPs {}, ToLookForIPs {} ",toTestHostIPs,toLookForHostIPs);
|
||||
for(InetAddress toTestIP:toTestHostIPs) {
|
||||
for(InetAddress toLookForIP:toLookForHostIPs)
|
||||
if(toTestIP.equals(toLookForIP)) return true;
|
||||
}
|
||||
}
|
||||
log.debug("HOSTS are different.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package org.gcube.data.transfer.plugins.thredds;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.gcube.common.resources.gcore.utils.XPathHelper;
|
||||
import org.w3c.dom.Element;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
public class TomcatSecurityHandler {
|
||||
|
||||
XPathHelper helper;
|
||||
|
||||
public TomcatSecurityHandler(String securityFile) throws SAXException, IOException {
|
||||
helper=CommonXML.getHelper(CommonXML.getDocument(new File(securityFile)));
|
||||
}
|
||||
|
||||
public String getThreddsAdminUser() {
|
||||
return getAdminUserElement().getAttribute("username");
|
||||
}
|
||||
|
||||
public String getThreddsAdminPassword() {
|
||||
return getAdminUserElement().getAttribute("password");
|
||||
}
|
||||
|
||||
public Element getAdminUserElement() {
|
||||
return (Element) helper.evaluateForNodes("//*[local-name()='user'][contains(@roles,'tdsConfig')]").item(0);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
package org.gcube.data.transfer.plugins.thredds;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashSet;
|
||||
|
||||
import javax.xml.transform.TransformerException;
|
||||
|
||||
import org.gcube.common.resources.gcore.utils.XPathHelper;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.CatalogCollection;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.DataSetRoot;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.DataSetScan;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class XMLCatalogHandler {
|
||||
|
||||
/*
|
||||
* NB XML STRUCTURE IS AS FOLLOW
|
||||
* catalog
|
||||
* |-datasetRoot?
|
||||
* |-datasetScan*
|
||||
* |-dataset [dedicated_catalogs, only in main catalog file]
|
||||
* |-catalogRef
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
|
||||
private static final String DEDICATED_CATALOGS_DATASET_ID="VRE_Catalogs";
|
||||
|
||||
private static final String LINKED_CATALOGS_XPATH="//*[local-name()='catalogRef' and parent::node()[local-name()='dataset']]";
|
||||
private static final String DECLARED_DATASETSCANS="//*[local-name()='datasetScan']";
|
||||
private static final String DECLARED_DATASETROOT="//*[local-name()='datasetRoot']";
|
||||
// private static final String ROOT_CATALOG_XPATH="/[local-name()='catalog']";
|
||||
private static final String CATALOG_COLLECTION_XPATH="//*[local-name()='dataset'][child::node()[local-name()='catalogRef']]|//*[@ID='"+DEDICATED_CATALOGS_DATASET_ID+"']";
|
||||
|
||||
//PARAMETRIC
|
||||
private static final String CATALOG_REFERENCE_BY_FILE="//*[local-name()='catalogRef'][@xlink:href='%s']";
|
||||
private static final String ELEMENT_BY_ID="//*[@ID='%s']";
|
||||
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* 1 - to expose reference
|
||||
* 2 - path to catalog file
|
||||
* 3 - ID
|
||||
*/
|
||||
private static final String catalogReferenceXMLPiece="<catalogRef "
|
||||
+"xmlns=\"http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0\"\n" +
|
||||
" xmlns:xlink=\"http://www.w3.org/1999/xlink\" "
|
||||
+ "xlink:title=\"%1$s\" xlink:href=\"%2$s\" ID=\"%3$s\" name=\"%1$s\"/>";
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private XPathHelper helper;
|
||||
private Document document;
|
||||
// private Element documentNode;
|
||||
private File currentCatalogFile;
|
||||
|
||||
|
||||
public XMLCatalogHandler(File catalog) throws SAXException, IOException {
|
||||
this.currentCatalogFile=catalog;
|
||||
document=CommonXML.getDocument(catalog);
|
||||
helper=CommonXML.getHelper(document);
|
||||
}
|
||||
|
||||
public void registerCatalog(File toRegisterCatalogFile,String toRegisterReference) throws SAXException, IOException {
|
||||
String filename=toRegisterCatalogFile.getName();
|
||||
String ID=filename.contains(".")?filename.substring(0, filename.lastIndexOf('.')):filename;
|
||||
|
||||
String toSetString=String.format(catalogReferenceXMLPiece, toRegisterReference,filename,ID);
|
||||
log.info("Checking if file is already referenced..");
|
||||
|
||||
log.debug("Checking by filname {} ",filename); //Check filename presence
|
||||
String referenceByFilenameXPATH=String.format(CATALOG_REFERENCE_BY_FILE,filename);
|
||||
NodeList filenameReferencesNodelist=helper.evaluateForNodes(referenceByFilenameXPATH);
|
||||
if(filenameReferencesNodelist.getLength()>0) {
|
||||
log.info("Filename {} is already declared. Updateing reference..",filename);
|
||||
CommonXML.addContent(referenceByFilenameXPATH,document,toSetString,helper,CommonXML.Position.replace);
|
||||
}else {
|
||||
|
||||
log.debug("Checking by ID {} ",ID); //else check by ID
|
||||
String referenceByIDXPATH=String.format(ELEMENT_BY_ID, ID);
|
||||
NodeList IDReferencesNodelist=helper.evaluateForNodes(referenceByIDXPATH);
|
||||
if(IDReferencesNodelist.getLength()>0) {
|
||||
log.info("ID {} found. Updateing reference..",ID);
|
||||
CommonXML.addContent(referenceByIDXPATH,document,toSetString,helper,CommonXML.Position.replace);
|
||||
}else {
|
||||
|
||||
log.info("No similar entries found. Adding reference.."); //else add
|
||||
String catalogCollectionXPATH=String.format(ELEMENT_BY_ID, DEDICATED_CATALOGS_DATASET_ID);
|
||||
CommonXML.addContent(catalogCollectionXPATH,document,toSetString,helper,CommonXML.Position.last_child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void close() throws IOException, TransformerException {
|
||||
CommonXML.writeOut(document, currentCatalogFile);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Parses the current catalog file and linked ones in order to
|
||||
* gather information on declared datasets
|
||||
*
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public ThreddsCatalog getCatalogDescriptor() {
|
||||
log.debug("loading catalogs from {} ",currentCatalogFile.getAbsolutePath());
|
||||
ThreddsCatalog toReturn=new ThreddsCatalog();
|
||||
toReturn.setCatalogFile(currentCatalogFile.getName());
|
||||
log.debug("Checking declared datasets in {} ",currentCatalogFile.getAbsolutePath());
|
||||
//get dataset root
|
||||
NodeList datasetRootNodes=helper.evaluateForNodes(DECLARED_DATASETROOT);
|
||||
if(datasetRootNodes.getLength()>0) {
|
||||
Element rootNode=(Element) datasetRootNodes.item(0);
|
||||
DataSetRoot root=new DataSetRoot();
|
||||
root.setLocation(rootNode.getAttribute("location"));
|
||||
root.setPath(rootNode.getAttribute("path"));
|
||||
toReturn.setDeclaredDataSetRoot(root);
|
||||
}
|
||||
//get dataset Scans
|
||||
NodeList datasetScans=helper.evaluateForNodes(DECLARED_DATASETSCANS);
|
||||
toReturn.setDeclaredDataSetScan(new HashSet<DataSetScan>());
|
||||
for(int i=0;i<datasetScans.getLength();i++) {
|
||||
Element scanNode=(Element) datasetScans.item(i);
|
||||
DataSetScan scan=new DataSetScan();
|
||||
scan.setID(scanNode.getAttribute("ID"));
|
||||
scan.setLocation(scanNode.getAttribute("location"));
|
||||
scan.setName(scanNode.getAttribute("name"));
|
||||
scan.setPath(scanNode.getAttribute("path"));
|
||||
toReturn.getDeclaredDataSetScan().add(scan);
|
||||
}
|
||||
|
||||
//Check for declared catalog collections
|
||||
NodeList catalogCollectionNodes=helper.evaluateForNodes(CATALOG_COLLECTION_XPATH);
|
||||
if(catalogCollectionNodes.getLength()>0) {
|
||||
Element catalogCollectionNode=(Element) catalogCollectionNodes.item(0);
|
||||
CatalogCollection collection=new CatalogCollection();
|
||||
|
||||
collection.setID(catalogCollectionNode.getAttribute("ID"));
|
||||
collection.setName(catalogCollectionNode.getAttribute("name"));
|
||||
log.debug("Found catalog collection ID {} , NAME {} ",collection.getID(),collection.getName());
|
||||
|
||||
collection.setLinkedCatalogs(new HashSet<ThreddsCatalog>());
|
||||
NodeList linkedCatalogsNodelist=helper.evaluateForNodes(LINKED_CATALOGS_XPATH);
|
||||
//load linked catalogs
|
||||
for(int i=0;i<linkedCatalogsNodelist.getLength();i++) {
|
||||
Element linkedCatalogElement=(Element) linkedCatalogsNodelist.item(i);
|
||||
String linkedCatalogFile=linkedCatalogElement.getAttribute("xlink:href");
|
||||
String linkedCatalogName=linkedCatalogElement.getAttribute("name");
|
||||
String linkedCatalogTitle=linkedCatalogElement.getAttribute("xlink:title");
|
||||
String linkedCatalogID=linkedCatalogElement.getAttribute("ID");
|
||||
try{
|
||||
ThreddsCatalog linkedCatalog=new XMLCatalogHandler(new File(currentCatalogFile.getParentFile(),linkedCatalogFile)).getCatalogDescriptor();
|
||||
linkedCatalog.setCatalogFile(linkedCatalogFile);
|
||||
linkedCatalog.setName(linkedCatalogName);
|
||||
linkedCatalog.setTitle(linkedCatalogTitle);
|
||||
linkedCatalog.setID(linkedCatalogID);
|
||||
collection.getLinkedCatalogs().add(linkedCatalog);
|
||||
}catch(Throwable t) {
|
||||
log.warn("Unable to parse {} ",linkedCatalogFile,t);
|
||||
}
|
||||
}
|
||||
toReturn.setSubCatalogs(collection);
|
||||
}
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
//**********************
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package org.gcube.data.transfer.plugins.thredds.catalog;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Map;
|
||||
|
||||
import org.gcube.data.transfer.model.ExecutionReport;
|
||||
import org.gcube.data.transfer.model.ExecutionReport.ExecutionReportFlag;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.gcube.data.transfer.plugin.AbstractPlugin;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginCleanupException;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginExecutionException;
|
||||
import org.gcube.data.transfer.plugins.thredds.ThreddsInstanceManager;
|
||||
import org.gcube.data.transfer.plugins.thredds.XMLCatalogHandler;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
public class RegisterCatalogPlugin extends AbstractPlugin{
|
||||
|
||||
private ThreddsInstanceManager instanceManager;
|
||||
|
||||
public RegisterCatalogPlugin(PluginInvocation invocation, ThreddsInstanceManager instanceManager) {
|
||||
super(invocation);
|
||||
this.instanceManager=instanceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cleanup() throws PluginCleanupException {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExecutionReport run() throws PluginExecutionException {
|
||||
try {
|
||||
Map<String,String> params=invocation.getParameters();
|
||||
String catalogFile=params.get(RegisterCatalogPluginFactory.CATALOG_FILE);
|
||||
if(catalogFile.contains(" ")) throw new PluginExecutionException("Invalid catalog filename "+catalogFile);
|
||||
String catalogReference=params.get(RegisterCatalogPluginFactory.CATALOG_REFERENCE);
|
||||
log.trace("Registering {} as {}",catalogFile,catalogReference);
|
||||
|
||||
XMLCatalogHandler handler=instanceManager.mainCatalogHandler();
|
||||
handler.registerCatalog(new File(catalogFile),catalogReference);
|
||||
handler.close();
|
||||
instanceManager.clearCache();
|
||||
return new ExecutionReport(invocation,"Registered catalog entry "+catalogReference,ExecutionReportFlag.SUCCESS);
|
||||
}catch(Throwable t) {
|
||||
log.error("Unable to register catalog.",t);
|
||||
throw new PluginExecutionException("Unable to register catalog.",t);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
package org.gcube.data.transfer.plugins.thredds.catalog;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
|
||||
import org.gcube.data.transfer.plugin.AbstractPluginFactory;
|
||||
import org.gcube.data.transfer.plugin.fails.ParameterException;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginExecutionException;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginInitializationException;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginShutDownException;
|
||||
import org.gcube.data.transfer.plugin.model.DataTransferContext;
|
||||
import org.gcube.data.transfer.plugins.thredds.ThreddsInstanceManager;
|
||||
import org.gcube.data.transfer.plugins.thredds.XMLCatalogHandler;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class RegisterCatalogPluginFactory extends AbstractPluginFactory<RegisterCatalogPlugin>{
|
||||
|
||||
static final String PLUGIN_ID="REGISTER_CATALOG";
|
||||
public static final String CATALOG_FILE="CATALOG_FILE";
|
||||
public static final String CATALOG_REFERENCE="CATALOG_REFERENCE";
|
||||
|
||||
|
||||
static final Map<String,String> PARAMETERS_DESCRIPTION= new HashMap<String,String>();
|
||||
|
||||
static{
|
||||
PARAMETERS_DESCRIPTION.put(CATALOG_REFERENCE, "[String value] The reference value under which the passed catalog should be registered.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public PluginInvocation checkInvocation(PluginInvocation arg0, String toRegisterCatalog) throws ParameterException {
|
||||
log.trace("Checking invocation {} ",arg0);
|
||||
Map<String,String> params=arg0.getParameters();
|
||||
if(params==null||params.isEmpty()||(!params.containsKey(CATALOG_REFERENCE))||(params.get(CATALOG_REFERENCE)==null))
|
||||
throw new ParameterException(CATALOG_REFERENCE + "parameter is mandatory.");
|
||||
arg0.getParameters().put(CATALOG_FILE, toRegisterCatalog);
|
||||
log.debug("Checking if file {} is a valid catalog..");
|
||||
try {
|
||||
new XMLCatalogHandler(new File(toRegisterCatalog)).getCatalogDescriptor();
|
||||
}catch(Throwable t) {
|
||||
throw new ParameterException("Catalog File "+toRegisterCatalog+" is not valid.", t);
|
||||
}
|
||||
|
||||
|
||||
return arg0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RegisterCatalogPlugin createWorker(PluginInvocation arg0) {
|
||||
return new RegisterCatalogPlugin(arg0,ThreddsInstanceManager.get(ctx));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return String.format("Registers a catalog file as <%s> under main thredds catalog.", CATALOG_REFERENCE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getID() {
|
||||
return PLUGIN_ID;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> getParameters() {
|
||||
return PARAMETERS_DESCRIPTION;
|
||||
}
|
||||
|
||||
DataTransferContext ctx=null;
|
||||
|
||||
@Override
|
||||
public boolean init(DataTransferContext arg0) throws PluginInitializationException {
|
||||
try{
|
||||
this.ctx=arg0;
|
||||
ThreddsInstanceManager.get(ctx).updatePublishedInfo();
|
||||
return true;
|
||||
}catch(Throwable t) {
|
||||
throw new PluginInitializationException(t);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean shutDown() throws PluginShutDownException {
|
||||
// TODO Auto-generated method stub
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getInfo() throws PluginExecutionException{
|
||||
try{
|
||||
ThreddsInfo info=ThreddsInstanceManager.get(ctx).getInfo();
|
||||
log.info("Returning {} ",info);
|
||||
return info;
|
||||
}catch(Throwable t) {
|
||||
log.error("Unable to gather catalog information",t);
|
||||
throw new PluginExecutionException("Unable to gather information",t);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.gcube.data.transfer.plugins.sis;
|
||||
package org.gcube.data.transfer.plugins.thredds.sis;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -10,52 +10,56 @@ import org.gcube.data.transfer.plugin.fails.ParameterException;
|
|||
import org.gcube.data.transfer.plugin.fails.PluginInitializationException;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginShutDownException;
|
||||
import org.gcube.data.transfer.plugin.model.DataTransferContext;
|
||||
import org.gcube.smartgears.configuration.application.ApplicationConfiguration;
|
||||
import org.gcube.data.transfer.plugins.thredds.ThreddsInstanceManager;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class SISPluginFactory extends AbstractPluginFactory<SisPlugin> {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static final String PLUGIN_ID="SIS/GEOTK";
|
||||
|
||||
public static final String SOURCE_PARAMETER="SOURCE_FILE";
|
||||
public static final String GEONETWORK_CATEGORY="GEONETWORK_CATEGORY";
|
||||
public static final String GEONETWORK_STYLESHEET="GEONETWORK_STYLESHEET";
|
||||
|
||||
public static final String VALIDATE_PARAMETER="VALIDATE";
|
||||
|
||||
|
||||
static final Map<String,String> PARAMETERS_DESCRIPTION= new HashMap<String,String>();
|
||||
|
||||
static{
|
||||
PARAMETERS_DESCRIPTION.put(GEONETWORK_CATEGORY, "[String value] GeoNetwork category for publiehd metadata. Default is 'Dataset'.");
|
||||
PARAMETERS_DESCRIPTION.put(GEONETWORK_STYLESHEET, "[String value] GeoNetwork stylesheet for publiehd metadata. Default is '_none_'.");
|
||||
PARAMETERS_DESCRIPTION.put(VALIDATE_PARAMETER, "[Boolean value] Ask GeoNetwork to validate generated metadata. Default is 'true'.");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
//INSTANCE
|
||||
public SISPluginFactory() {
|
||||
|
||||
|
||||
}
|
||||
|
||||
String publicCatalogLocation=null;
|
||||
|
||||
|
||||
|
||||
DataTransferContext ctx;
|
||||
|
||||
|
||||
@Override
|
||||
public PluginInvocation checkInvocation(PluginInvocation arg0,String transferredFile) throws ParameterException {
|
||||
log.debug("Setting default parameters for {} ",arg0);
|
||||
Map<String,String> params=arg0.getParameters();
|
||||
|
||||
|
||||
if(params==null||params.isEmpty()) params=new HashMap<String,String>();
|
||||
if(!params.containsKey(SOURCE_PARAMETER)) params.put(SOURCE_PARAMETER, transferredFile);
|
||||
if(!params.containsKey(GEONETWORK_CATEGORY)) params.put(GEONETWORK_CATEGORY, "Dataset");
|
||||
if(!params.containsKey(GEONETWORK_STYLESHEET)) params.put(GEONETWORK_STYLESHEET, "_none_");
|
||||
String source=params.get(SOURCE_PARAMETER);
|
||||
log.debug("Checking access to source {} ",source);
|
||||
|
||||
|
||||
if(source==null||source.length()==0) throw new ParameterException(SOURCE_PARAMETER+" cannot be null.");
|
||||
try{
|
||||
File f=new File(source);
|
||||
|
|
@ -66,15 +70,15 @@ public class SISPluginFactory extends AbstractPluginFactory<SisPlugin> {
|
|||
}catch(Exception e){
|
||||
throw new ParameterException("Unable to access source file ",e);
|
||||
}
|
||||
|
||||
|
||||
arg0.setParameters(params);
|
||||
return arg0;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public SisPlugin createWorker(PluginInvocation arg0) {
|
||||
return new SisPlugin(arg0,ctx,publicCatalogLocation);
|
||||
return new SisPlugin(arg0,ThreddsInstanceManager.get(ctx));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -94,26 +98,13 @@ public class SISPluginFactory extends AbstractPluginFactory<SisPlugin> {
|
|||
|
||||
@Override
|
||||
public boolean init(DataTransferContext ctx) throws PluginInitializationException {
|
||||
log.trace("Loading configuration .. ");
|
||||
this.ctx=ctx;
|
||||
String threddsContentRoot=null;
|
||||
// String threddsContentRoot=System.getenv("THREDDS_CONTENT_ROOT")+File.separator+"thredds";
|
||||
// log.info("Thredds catalog base path from ENV is {} ",threddsContentRoot);
|
||||
|
||||
for(ApplicationConfiguration app:ctx.getCtx().container().configuration().apps()) {
|
||||
log.debug("Found app {} ",app.context());
|
||||
if(app.context().equals("thredds")||app.context().equals("/thredds")) {
|
||||
threddsContentRoot=app.persistence().location();
|
||||
log.info("Thredds catalog base path from Context is {} ",threddsContentRoot);
|
||||
|
||||
// Found thredds
|
||||
// Get catalog base folder
|
||||
// app.persistence().
|
||||
}
|
||||
try {
|
||||
this.ctx=ctx;
|
||||
ThreddsInstanceManager.get(ctx).updatePublishedInfo();
|
||||
return true;
|
||||
}catch(Throwable t) {
|
||||
throw new PluginInitializationException(t);
|
||||
}
|
||||
if(threddsContentRoot==null) throw new PluginInitializationException("No Thredds instance found in context");
|
||||
publicCatalogLocation=threddsContentRoot+"/public/netcdf/";
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -121,7 +112,6 @@ public class SISPluginFactory extends AbstractPluginFactory<SisPlugin> {
|
|||
return true;
|
||||
}
|
||||
|
||||
|
||||
private DataTransferContext ctx;
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.gcube.data.transfer.plugins.sis;
|
||||
package org.gcube.data.transfer.plugins.thredds.sis;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Map;
|
||||
|
|
@ -11,10 +11,12 @@ import org.apache.sis.xml.XML;
|
|||
import org.gcube.data.transfer.model.ExecutionReport;
|
||||
import org.gcube.data.transfer.model.ExecutionReport.ExecutionReportFlag;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.DataSet;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
|
||||
import org.gcube.data.transfer.plugin.AbstractPlugin;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginCleanupException;
|
||||
import org.gcube.data.transfer.plugin.fails.PluginExecutionException;
|
||||
import org.gcube.data.transfer.plugin.model.DataTransferContext;
|
||||
import org.gcube.data.transfer.plugins.thredds.ThreddsInstanceManager;
|
||||
import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
|
||||
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
|
||||
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocation;
|
||||
|
|
@ -28,12 +30,12 @@ import lombok.extern.slf4j.Slf4j;
|
|||
public class SisPlugin extends AbstractPlugin {
|
||||
|
||||
File tmp=null;
|
||||
DataTransferContext ctx;
|
||||
String publicCatalogLocation;
|
||||
public SisPlugin(PluginInvocation invocation, DataTransferContext ctx, String publicCatalogLocation) {
|
||||
|
||||
ThreddsInstanceManager instanceManager;
|
||||
|
||||
public SisPlugin(PluginInvocation invocation, ThreddsInstanceManager instanceManager) {
|
||||
super(invocation);
|
||||
this.ctx=ctx;
|
||||
this.publicCatalogLocation=publicCatalogLocation;
|
||||
this.instanceManager=instanceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -53,6 +55,16 @@ public class SisPlugin extends AbstractPlugin {
|
|||
String dataStorePath=params.get(SISPluginFactory.SOURCE_PARAMETER);
|
||||
String category=params.get(SISPluginFactory.GEONETWORK_CATEGORY);
|
||||
String stylesheet=params.get(SISPluginFactory.GEONETWORK_STYLESHEET);
|
||||
|
||||
Boolean validateMetadata=true;
|
||||
|
||||
try {
|
||||
if(params.containsKey(SISPluginFactory.VALIDATE_PARAMETER))
|
||||
validateMetadata=Boolean.parseBoolean(params.get(SISPluginFactory.VALIDATE_PARAMETER));
|
||||
}catch(Throwable t) {
|
||||
log.warn("Unable to read validate parameter : "+params.get(SISPluginFactory.VALIDATE_PARAMETER),t);
|
||||
}
|
||||
|
||||
File dataStore=new File(dataStorePath);
|
||||
log.debug("Extracting meta from {} ",dataStore.getAbsolutePath());
|
||||
Metadata meta=getMetaFromFile(dataStore);
|
||||
|
|
@ -63,39 +75,32 @@ public class SisPlugin extends AbstractPlugin {
|
|||
|
||||
org.gcube.spatial.data.sdi.interfaces.Metadata metadataInterface=SDIAbstractPlugin.metadata().build();
|
||||
|
||||
String hostname=ctx.getCtx().container().configuration().hostname();
|
||||
String filename=dataStore.getName();
|
||||
String catalog=getCatalogFromPath(dataStorePath);
|
||||
String hostname=instanceManager.getCurrentHostname();
|
||||
String filename=dataStore.getName();
|
||||
ThreddsCatalog catalog=instanceManager.getInfo().getCatalogByFittingLocation(dataStorePath);
|
||||
log.debug("Catalog for transferred File at {} is {} ",dataStorePath,catalog);
|
||||
DataSet catalogDataset=catalog.getDataSetFromLocation(dataStorePath);
|
||||
|
||||
String datasetSubPath=dataStorePath.substring(catalogDataset.getLocation().length(), dataStorePath.lastIndexOf("/"));
|
||||
String datasetPath=catalogDataset.getPath()+datasetSubPath;
|
||||
|
||||
|
||||
log.info("Publishing dataset {} with path {} ",dataStorePath,datasetPath);
|
||||
Set<TemplateInvocation> invocations=new TemplateInvocationBuilder().
|
||||
threddsOnlineResources(hostname,filename, catalog).get();
|
||||
threddsOnlineResources(hostname,filename, datasetPath).get();
|
||||
|
||||
MetadataPublishOptions options=new MetadataPublishOptions(invocations);
|
||||
options.setGeonetworkCategory(category);
|
||||
options.setGeonetworkStyleSheet(stylesheet);
|
||||
options.setValidate(false);
|
||||
options.setMakePublic(catalogDataset.getPath().startsWith("public"));
|
||||
|
||||
options.setValidate(validateMetadata);
|
||||
|
||||
|
||||
MetadataReport report=metadataInterface.pushMetadata(tmp, options);
|
||||
log.debug("Metadata report is {} ",report);
|
||||
return new ExecutionReport(invocation,
|
||||
"Published/Updated meta with id : "+report.getPublishedID()+" , UUID "+report.getPublishedUUID(),ExecutionReportFlag.SUCCESS);
|
||||
|
||||
//// TemplateManager.getTHREDDSLinks(new ThreddsLinkRequest(hostname, filename, catalog, gisViewerLink))
|
||||
// // TODO Info from infrastructure
|
||||
// try{
|
||||
// long id=publishMetadata(tmp, category, stylesheet,getGNPublisher());
|
||||
// return new ExecutionReport(invocation, "Published meta with id : "+id, ExecutionReportFlag.SUCCESS);
|
||||
// }catch(GNServerException e){
|
||||
// //Fail can be due to already existing uuid.
|
||||
// log.debug("Trying to update existing uuid");
|
||||
//
|
||||
// MetadataHandler metaHandler=new MetadataHandler(tmp);
|
||||
// String uuid=metaHandler.getUUID();
|
||||
// long id= updateMetadata(tmp, uuid, category, stylesheet,getGNPublisher());
|
||||
// return new ExecutionReport(invocation, "Updated meta with id : "+id+", UUID : "+uuid, ExecutionReportFlag.SUCCESS);
|
||||
// }
|
||||
|
||||
|
||||
|
||||
}catch(DataStoreException e){
|
||||
|
|
@ -106,55 +111,11 @@ public class SisPlugin extends AbstractPlugin {
|
|||
throw new PluginExecutionException("Unexpected error while generating meta.",t);
|
||||
}
|
||||
}
|
||||
|
||||
// public GeoNetworkPublisher getGNPublisher() throws Exception{
|
||||
// if(publisher==null){
|
||||
// publisher=GeoNetwork.get();
|
||||
// publisher.login(LoginLevel.DEFAULT);
|
||||
// }
|
||||
// return publisher;
|
||||
// }
|
||||
//
|
||||
|
||||
public static final Metadata getMetaFromFile(Object dataStore) throws UnsupportedStorageException, DataStoreException{
|
||||
return DataStores.open(dataStore).getMetadata();
|
||||
}
|
||||
|
||||
private String getCatalogFromPath(String path){
|
||||
return getCatalogFromPath(path, publicCatalogLocation);
|
||||
}
|
||||
|
||||
public static String getCatalogFromPath(String path, String publicCatalogLocation) {
|
||||
log.debug("Getting catalog from path {} ",path);
|
||||
String catalog=path.substring(publicCatalogLocation.length());
|
||||
if(catalog.contains("/")) {
|
||||
catalog=catalog.substring(0, catalog.lastIndexOf("/"));
|
||||
log.debug("Subcatalog found {} ",catalog);
|
||||
}else {
|
||||
log.debug("No catalog found");
|
||||
catalog=null;
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
|
||||
//
|
||||
// public long publishMetadata(File toPublish,String category, String stylesheet, GeoNetworkPublisher publisher) throws Exception{
|
||||
// GNInsertConfiguration config=publisher.getCurrentUserConfiguration(category, stylesheet);
|
||||
// config.setValidate(false);
|
||||
// long toReturn= publisher.insertMetadata(config, toPublish);
|
||||
// return toReturn;
|
||||
// }
|
||||
//
|
||||
// public long updateMetadata(File toPublish,String uuid,String category,String styleSheet, GeoNetworkPublisher publisher)throws Exception{
|
||||
// long id=0l;
|
||||
// if(publisher.getConfiguration().getGeoNetworkVersion().equals(Version.DUE)){
|
||||
// GNSearchRequest req=new GNSearchRequest();
|
||||
// req.addParam(GNSearchRequest.Param.any,uuid);
|
||||
//
|
||||
// GNSearchResponse resp=publisher.query(req);
|
||||
// id=resp.iterator().next().getId();
|
||||
// }else id=publisher.getInfo(uuid).getId();
|
||||
// publisher.updateMetadata(id, toPublish);
|
||||
// return id;
|
||||
// }
|
||||
}
|
||||
|
|
@ -1 +1,2 @@
|
|||
org.gcube.data.transfer.plugins.sis.SISPluginFactory
|
||||
org.gcube.data.transfer.plugins.thredds.sis.SISPluginFactory
|
||||
org.gcube.data.transfer.plugins.thredds.catalog.RegisterCatalogPluginFactory
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
th.se.category=Gis
|
||||
th.se.platform=Thredds
|
||||
th.se.remoteManagement.access=Remote_Management
|
||||
|
||||
#IS
|
||||
is.registration.timeout=60000
|
||||
|
||||
#Context
|
||||
context.loading.timeout=600000
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package org.gcube.data.transfer.plugins.sis;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.xml.transform.TransformerException;
|
||||
|
||||
import org.gcube.data.transfer.plugins.thredds.XMLCatalogHandler;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
public class TestAddCatalog {
|
||||
|
||||
public static void main(String[] args) throws SAXException, IOException, TransformerException {
|
||||
String mainCatalogPath="/home/fabio/Desktop/catalog.xml";
|
||||
XMLCatalogHandler handler=new XMLCatalogHandler(new File(mainCatalogPath));
|
||||
System.out.println("Catalogs : "+handler.getCatalogDescriptor());
|
||||
handler.registerCatalog(new File("/home/fabio/Desktop/Tuna_Atlas_VRE_catalog.xml"), "testReplace2");
|
||||
handler.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,11 +1,32 @@
|
|||
package org.gcube.data.transfer.plugins.sis;
|
||||
|
||||
import javax.ws.rs.client.Client;
|
||||
import javax.ws.rs.client.ClientBuilder;
|
||||
import javax.ws.rs.client.WebTarget;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.data.transfer.library.client.AuthorizationFilter;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.DataSet;
|
||||
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
|
||||
import org.glassfish.jersey.client.ClientConfig;
|
||||
|
||||
public class TestGetCatalog {
|
||||
|
||||
public static void main(String[] args) {
|
||||
String path="/home/gcube/tomcat/content/thredds/public/netcdf/myCatalog/CERSAT-GLO-CLIM_WIND_L4-OBS_FULL_TIME_SERIE_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_1366217956317.nc";
|
||||
String publicLocation="/home/gcube/tomcat/content/thredds/public/netcdf/";
|
||||
System.out.println(SisPlugin.getCatalogFromPath(path, publicLocation));
|
||||
SecurityTokenProvider.instance.set("f851ba11-bd3e-417a-b2c2-753b02bac506-98187548");
|
||||
String dataStorePath="/data/content/thredds/devVRE/indian_ocean_catch_5deg_1m_1952_11_01_2016_01_01_tunaatlasIRD_level1.nc";
|
||||
//getting threddsInfo from thredds-d
|
||||
Client client=ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class));
|
||||
WebTarget target=client.target(String.format("https://%s/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG", "thredds-d-d4s.d4science.org"));
|
||||
ThreddsInfo info=target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class);
|
||||
System.out.println(info.getCatalogByFittingLocation(dataStorePath));
|
||||
DataSet catalogDataset=info.getDataSetFromLocation(dataStorePath);
|
||||
|
||||
|
||||
String datasetSubPath=dataStorePath.substring(catalogDataset.getLocation().length(), dataStorePath.lastIndexOf("/"));
|
||||
String datasetPath=catalogDataset.getPath()+datasetSubPath;
|
||||
System.out.println("Path : "+datasetPath);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import java.security.NoSuchAlgorithmException;
|
|||
import org.apache.sis.storage.DataStoreException;
|
||||
import org.apache.sis.storage.UnsupportedStorageException;
|
||||
import org.gcube.data.transfer.model.TransferTicket.Status;
|
||||
import org.gcube.data.transfer.plugins.thredds.sis.SisPlugin;
|
||||
|
||||
public class TestGetMetadata {
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue