2015-09-23 11:53:42 +02:00
package org.gcube.dataanalysis.geo.utils ;
import java.util.List ;
2017-05-24 17:35:50 +02:00
import java.util.regex.Matcher ;
import java.util.regex.Pattern ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
import org.gcube.common.resources.gcore.GCoreEndpoint ;
import org.gcube.common.scope.api.ScopeProvider ;
import org.gcube.data.transfer.library.TransferResult ;
2015-09-23 11:53:42 +02:00
import org.gcube.dataanalysis.executor.util.DataTransferer ;
import org.gcube.dataanalysis.geo.connectors.netcdf.NetCDFDataExplorer ;
import org.gcube.dataanalysis.geo.infrastructure.GeoNetworkInspector ;
import org.gcube.dataanalysis.geo.meta.GenericLayerMetadata ;
import org.gcube.dataanalysis.geo.meta.OGCFormatter ;
2017-05-24 17:35:50 +02:00
import org.gcube.resources.discovery.client.api.DiscoveryClient ;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery ;
import org.gcube.resources.discovery.icclient.ICFactory ;
2015-09-23 11:53:42 +02:00
import org.opengis.metadata.identification.TopicCategory ;
2017-05-24 17:35:50 +02:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
2015-09-23 11:53:42 +02:00
import ucar.nc2.dt.GridDatatype ;
public class ThreddsPublisher {
2017-05-24 17:35:50 +02:00
private static final String threddsServiceName = " Thredds " ;
private static final String threddsServiceClass = " SDI " ;
private static final String dataTransferName = " data-transfer-service " ;
private static final String dataTransferClass = " DataTransfer " ;
private static final String dataTransferEndpoint = " org.gcube.data.transfer.service.DTService " ;
private static final String threddsPersistenceID = " thredds " ;
private static final String threddsRemoteFolder = " /public/netcdf " ;
private static final String threddsFileServerPath = " /thredds/fileServer/public/netcdf/ " ;
private static final String threddsCatalogPath = " /thredds/catalog/public/netcdf/catalog.xml " ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
private static final String netCDFExtension = " .nc " ;
private static final Logger log = LoggerFactory . getLogger ( ThreddsPublisher . class ) ;
public static boolean publishOnThredds ( String username , String fileAbsolutePath , String layerTitle , String layerName , String abstractField , String [ ] topics , double resolution , boolean isprivate ) throws Exception {
2015-09-23 11:53:42 +02:00
//TODO manage faults
2017-05-24 17:35:50 +02:00
DiscoveryClient < String > threddsClient = ICFactory . client ( ) ;
SimpleQuery thrredsQuery = ICFactory . queryFor ( GCoreEndpoint . class ) ;
thrredsQuery . addCondition ( String . format ( " $resource/Profile/ServiceName eq '%s' " , threddsServiceName ) ) ;
thrredsQuery . addCondition ( String . format ( " $resource/Profile/ServiceClass eq '%s' " , threddsServiceClass ) ) ;
thrredsQuery . setResult ( " $resource/Profile/GHN/@UniqueID/string() " ) ;
List < String > threddsAddress = threddsClient . submit ( thrredsQuery ) ;
2015-09-23 11:53:42 +02:00
if ( threddsAddress . size ( ) = = 0 )
2017-05-24 17:35:50 +02:00
throw new Exception ( " Thredds Endpoint not found in scope " + ScopeProvider . instance . get ( ) ) ;
String threddsWhnId = threddsAddress . get ( 0 ) ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
DiscoveryClient < String > dataTransferClient = ICFactory . client ( ) ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
SimpleQuery dataTransferQuery = ICFactory . queryFor ( GCoreEndpoint . class ) ;
dataTransferQuery . addCondition ( String . format ( " $resource/Profile/ServiceName/string() eq '%s' " , dataTransferName ) ) ;
dataTransferQuery . addCondition ( String . format ( " $resource/Profile/ServiceClass/string() eq '%s' " , dataTransferClass ) ) ;
dataTransferQuery . addCondition ( String . format ( " $resource/Profile/GHN/@UniqueID/string() eq '%s' " , threddsWhnId ) ) ;
dataTransferQuery . setResult ( String . format ( " $resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint[@EntryName/string() eq \" %s \" ]/string() " , dataTransferEndpoint ) ) ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
List < String > dataTransferAddress = dataTransferClient . submit ( dataTransferQuery ) ;
2015-09-23 11:53:42 +02:00
if ( dataTransferAddress . size ( ) = = 0 )
2017-05-24 17:35:50 +02:00
throw new Exception ( " Data Transfer services is not available in scope " + ScopeProvider . instance . get ( ) ) ;
String threddsDTService = dataTransferAddress . get ( 0 ) ;
log . debug ( " data transfer found is {} " , threddsDTService ) ;
Pattern pattern = Pattern . compile ( " (https?)://([^:/]*)(:( \\ d{2,5}))?.* " ) ;
Matcher matcher = pattern . matcher ( threddsDTService ) ;
if ( ! matcher . find ( ) )
throw new Exception ( " wrong address found " + threddsDTService ) ;
String dataTransferProtocol = matcher . group ( 1 ) ;
String dataTransferHost = matcher . group ( 2 ) ;
String portAsString = matcher . group ( 4 ) ;
Integer dataTransferPort = portAsString = = null ? null : Integer . parseInt ( portAsString ) ;
StringBuilder threedsBaseURL = new StringBuilder ( dataTransferProtocol ) . append ( " :// " ) . append ( dataTransferHost ) ;
if ( portAsString ! = null )
threedsBaseURL . append ( " : " ) . append ( portAsString ) ;
2015-09-23 11:53:42 +02:00
2016-04-05 16:00:10 +02:00
boolean gridded = true ;
2015-09-23 11:53:42 +02:00
if ( fileAbsolutePath . endsWith ( " .nc " ) ) {
2017-05-24 17:35:50 +02:00
log . debug ( " checking NetCDF file coherence {} " , fileAbsolutePath ) ;
2016-04-05 16:00:10 +02:00
//let's publish also if the netCDF is not gridded
try {
NetCDFDataExplorer . getGrid ( layerName , fileAbsolutePath ) ;
} catch ( Exception e ) {
gridded = false ;
2017-05-24 17:35:50 +02:00
log . debug ( " NetCDF is not gridded {} " , fileAbsolutePath ) ;
2016-04-05 16:00:10 +02:00
}
2015-09-23 11:53:42 +02:00
}
2017-05-24 17:35:50 +02:00
log . debug ( " Transferring via DT to {} with parameters {} {} {} {} " , threddsDTService , dataTransferHost , dataTransferPort , fileAbsolutePath , threddsRemoteFolder ) ;
TransferResult transferResult = DataTransferer . transferFileToService ( ScopeProvider . instance . get ( ) , username , dataTransferHost , dataTransferPort , fileAbsolutePath , threddsRemoteFolder , threddsPersistenceID ) ;
String realFileName = transferResult . getRemotePath ( ) . substring ( transferResult . getRemotePath ( ) . lastIndexOf ( " / " ) + 1 ) ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
log . debug ( " Adding metadata on GeoNetwork, real file name on threadds is {} " , realFileName ) ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
if ( fileAbsolutePath . endsWith ( netCDFExtension ) & & gridded )
publishNetCDFMeta ( ScopeProvider . instance . get ( ) , layerTitle , abstractField , realFileName , layerName , threedsBaseURL . toString ( ) , username , topics , isprivate ) ;
2015-09-23 11:53:42 +02:00
else {
2016-04-05 16:00:10 +02:00
if ( resolution = = - 1 & & gridded )
2015-09-23 11:53:42 +02:00
throw new Exception ( " Specify valid resolution parameter for non-NetCDF raster datasets " ) ;
2017-05-24 17:35:50 +02:00
publishOtherFileMeta ( ScopeProvider . instance . get ( ) , layerTitle , resolution , abstractField , realFileName , threedsBaseURL . toString ( ) , username , topics , isprivate ) ;
2015-09-23 11:53:42 +02:00
}
2017-05-24 17:35:50 +02:00
log . debug ( " Finished " ) ;
2015-09-23 11:53:42 +02:00
return true ;
}
2016-06-14 12:47:46 +02:00
private static void publishOtherFileMeta ( String scope , String layerTitle , double resolution , String abstractField , String filename , String threddsURL , String username , String [ ] topics , boolean isprivate ) throws Exception {
2015-09-23 11:53:42 +02:00
GenericLayerMetadata metadataInserter = new GenericLayerMetadata ( ) ;
GeoNetworkInspector gninspector = new GeoNetworkInspector ( ) ;
gninspector . setScope ( scope ) ;
String geonetworkURL = gninspector . getGeonetworkURLFromScope ( ) ;
String geonetworkUser = gninspector . getGeonetworkUserFromScope ( ) ;
String geonetworkPassword = gninspector . getGeonetworkPasswordFromScope ( ) ;
2016-06-14 12:47:46 +02:00
String geonetworkGroup = " " ;
if ( isprivate )
geonetworkGroup = gninspector . getGeonetworkPrivateGroup ( ) ;
else
geonetworkGroup = gninspector . getGeonetworkPublicGroup ( ) ;
2017-05-24 17:35:50 +02:00
log . debug ( " GeoNetwork Info: " + geonetworkURL + " " + geonetworkUser + " " + geonetworkGroup ) ;
2015-09-23 11:53:42 +02:00
metadataInserter . setGeonetworkUrl ( geonetworkURL ) ;
metadataInserter . setGeonetworkPwd ( geonetworkPassword ) ;
metadataInserter . setGeonetworkUser ( geonetworkUser ) ;
2016-06-14 12:47:46 +02:00
metadataInserter . setGeonetworkGroup ( geonetworkGroup ) ;
2015-09-23 11:53:42 +02:00
metadataInserter . setTitle ( layerTitle ) ;
metadataInserter . setCategoryTypes ( " _ " + TopicCategory . ENVIRONMENT . name ( ) + " _ " ) ;
metadataInserter . setAbstractField ( abstractField + " Hosted on the D4Science Thredds Catalog: " + threddsURL ) ;
metadataInserter . setCustomTopics ( topics ) ;
metadataInserter . setAuthor ( username ) ;
metadataInserter . setResolution ( resolution ) ;
2017-05-24 17:35:50 +02:00
log . debug ( " Res: " + resolution ) ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
String [ ] urls = { threddsURL + threddsFileServerPath + filename } ;
2015-09-23 11:53:42 +02:00
String [ ] protocols = { " HTTP " } ;
2017-05-24 17:35:50 +02:00
log . debug ( " Publishing in group: " + metadataInserter . getGeonetworkGroup ( ) ) ;
log . debug ( " Inserting custom metadata " ) ;
2016-09-27 18:00:11 +02:00
metadataInserter . customMetaDataInsert ( urls , protocols , isprivate ) ;
2015-09-23 11:53:42 +02:00
}
2017-05-24 17:35:50 +02:00
private static void publishNetCDFMeta ( String scope , String layerTitle , String abstractField , String filename , String netCDFLayerName , String threddsURL , String username , String [ ] topics , boolean isprivate ) throws Exception {
log . debug ( " Getting GeoNetwork Info " ) ;
2015-09-23 11:53:42 +02:00
GenericLayerMetadata metadataInserter = new GenericLayerMetadata ( ) ;
GeoNetworkInspector gninspector = new GeoNetworkInspector ( ) ;
gninspector . setScope ( scope ) ;
String geonetworkURL = gninspector . getGeonetworkURLFromScope ( ) ;
String geonetworkUser = gninspector . getGeonetworkUserFromScope ( ) ;
String geonetworkPassword = gninspector . getGeonetworkPasswordFromScope ( ) ;
2016-06-14 12:47:46 +02:00
String geonetworkGroup = " " ;
if ( isprivate )
geonetworkGroup = gninspector . getGeonetworkPrivateGroup ( ) ;
else
geonetworkGroup = gninspector . getGeonetworkPublicGroup ( ) ;
2015-09-23 11:53:42 +02:00
2017-05-24 17:35:50 +02:00
log . debug ( " GeoNetwork Info: " + geonetworkURL + " " + geonetworkUser + " " + geonetworkGroup ) ;
2015-09-23 11:53:42 +02:00
metadataInserter . setGeonetworkUrl ( geonetworkURL ) ;
metadataInserter . setGeonetworkPwd ( geonetworkPassword ) ;
metadataInserter . setGeonetworkUser ( geonetworkUser ) ;
2016-06-14 12:47:46 +02:00
metadataInserter . setGeonetworkGroup ( geonetworkGroup ) ;
2015-09-23 11:53:42 +02:00
metadataInserter . setTitle ( layerTitle ) ;
metadataInserter . setCategoryTypes ( " _ " + TopicCategory . ENVIRONMENT . name ( ) + " _ " ) ;
metadataInserter . setAbstractField ( abstractField + " Hosted on the D4Science Thredds Catalog: " + threddsURL ) ;
metadataInserter . setCustomTopics ( topics ) ;
metadataInserter . setAuthor ( username ) ;
2017-05-24 17:35:50 +02:00
String threddscatalog = threddsURL + threddsCatalogPath ;
String url = OGCFormatter . getOpenDapURL ( threddscatalog , filename ) ;
log . debug ( " OpenDAP URL: {} " , url ) ;
2015-09-23 11:53:42 +02:00
GridDatatype gdt = NetCDFDataExplorer . getGrid ( netCDFLayerName , url ) ;
double minX = NetCDFDataExplorer . getMinX ( gdt . getCoordinateSystem ( ) ) ;
double maxX = NetCDFDataExplorer . getMaxX ( gdt . getCoordinateSystem ( ) ) ;
double minY = NetCDFDataExplorer . getMinY ( gdt . getCoordinateSystem ( ) ) ;
double maxY = NetCDFDataExplorer . getMaxY ( gdt . getCoordinateSystem ( ) ) ;
double resolutionY = NetCDFDataExplorer . getResolution ( netCDFLayerName , url ) ;
metadataInserter . setResolution ( resolutionY ) ;
2017-05-24 17:35:50 +02:00
log . debug ( " minX: " + minX + " minY: " + minY + " maxX: " + maxX + " maxY: " + maxY + " Res: " + resolutionY ) ;
2015-09-23 11:53:42 +02:00
String wms = OGCFormatter . getWmsNetCDFUrl ( url , netCDFLayerName , OGCFormatter . buildBoundingBox ( minX , minY , maxX , maxY ) ) . replace ( " width=676 " , " width=640 " ) . replace ( " height=330 " , " height=480 " ) ;
2017-05-24 17:35:50 +02:00
log . debug ( " WMS URL: {} " , wms ) ;
2015-09-23 11:53:42 +02:00
String wcs = OGCFormatter . getWcsNetCDFUrl ( url , netCDFLayerName , OGCFormatter . buildBoundingBox ( minX , minY , maxX , maxY ) ) . replace ( " width=676 " , " width=640 " ) . replace ( " height=330 " , " height=480 " ) ;
2017-05-24 17:35:50 +02:00
log . debug ( " WCS URL: {} " , wcs ) ;
String fileServerUrl = threddsURL + threddsFileServerPath + filename ;
log . debug ( " HTTP URL: {} " , fileServerUrl ) ;
String [ ] urls = { fileServerUrl , wms , wcs , url } ;
2015-09-23 11:53:42 +02:00
String [ ] protocols = { " HTTP " , " WMS " , " WCS " , " OPeNDAP " } ;
metadataInserter . setXLeftLow ( minX ) ;
metadataInserter . setYLeftLow ( minY ) ;
metadataInserter . setXRightUpper ( maxX ) ;
metadataInserter . setYRightUpper ( maxY ) ;
2017-05-24 17:35:50 +02:00
log . debug ( " Publishing in group: {} " , metadataInserter . getGeonetworkGroup ( ) ) ;
log . debug ( " Inserting metadata " ) ;
2016-09-27 18:00:11 +02:00
metadataInserter . customMetaDataInsert ( urls , protocols , isprivate ) ;
2015-09-23 11:53:42 +02:00
}
}