package org.gcube.application.geoportal.service.engine; import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod; import it.geosolutions.geoserver.rest.GeoServerRESTReader; import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; import it.geosolutions.geoserver.rest.decoder.RESTLayer; import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder; import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder; import it.geosolutions.geoserver.rest.encoder.feature.FeatureTypeAttribute; import it.geosolutions.geoserver.rest.encoder.feature.GSAttributeEncoder; import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.model.db.DBConstants; import org.gcube.application.geoportal.model.db.DatabaseConnection; import org.gcube.application.geoportal.model.db.PostgisTable; import org.gcube.application.geoportal.service.engine.mongo.PostgisIndex; import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; import org.gcube.common.storagehub.client.dsl.FileContainer; import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.library.faults.RemoteServiceException; import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.spatial.data.gis.GISInterface; import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; import java.io.File; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @Slf4j public class SDIManager { static private final String EPSG_4326="EPSG:4326"; static private final String WGS84_FULL="GEOGCS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", SPHEROID[\"WGS 84\", 6378137.0, 298.257223563, AUTHORITY[\"EPSG\",\"7030\"]],"+ "AUTHORITY[\"EPSG\",\"6326\"]], PRIMEM[\"Greenwich\", 0.0, AUTHORITY[\"EPSG\",\"8901\"]], UNIT[\"degree\", 0.017453292519943295],"+ "AXIS[\"Geodetic longitude\", EAST], AXIS[\"Geodetic latitude\", NORTH], AUTHORITY[\"EPSG\",\"4326\"]]"; public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+"); public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); private final GISInterface gis; @Getter private final DataTransferClient dtGeoServer; private final String geoserverHostName; private final AbstractGeoServerDescriptor currentGeoserver; public SDIManager() throws SDIInteractionException { try{ log.debug("Initializing GIS Interface.."); gis=GISInterface.get(); currentGeoserver=gis.getCurrentGeoServer(); if(currentGeoserver==null) throw new Exception("Unable to contact data transfer for geoserver "); log.debug("Found geoserver descriptor "+currentGeoserver); geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); dtGeoServer=DataTransferClient.getInstanceByEndpoint("http://"+geoserverHostName); if(!currentGeoserver.getReader().existGeoserver()) throw new Exception("Geoserver not reachable"); }catch(Exception e) { throw new SDIInteractionException("Unable to initialize SDI Manager",e); } } public RemoteFileDescriptor getGeoServerRemoteFolder() throws RemoteServiceException { return dtGeoServer.getWebClient().getInfo("geoserver/GNA"); } public String createWorkspace(String toCreate) throws SDIInteractionException { try { if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { log.debug("Creating workspace : "+toCreate); if(!currentGeoserver.getPublisher().createWorkspace(toCreate)) throw new SDIInteractionException("Unable to create workspace "+toCreate); }else log.debug("Workspace "+toCreate+" exists."); return toCreate; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create workspace "+toCreate,e); } } // GEOSERVER-PERSISTENCE-ID / GNA / PROJECT-ID/ LAYER-ID /FILENAME(no extension)/... public GeoServerContent pushShapeLayerFileSet(SDILayerDescriptor currentElement,String workspace, String projectId) throws SDIInteractionException{ try { // String remoteFolder=null; // String fileName=null; log.debug("Publishing "+currentElement+" files to geoserver @ "+geoserverHostName); GeoServerContent content=new GeoServerContent(); content.setGeoserverHostName(geoserverHostName); content.setWorkspace(workspace); WorkspaceManager wsManager=new WorkspaceManager(); currentElement.getActualContent().forEach((PersistedContent c)->{ try { if(c instanceof WorkspaceContent) { WorkspaceContent wc=(WorkspaceContent) c; FileContainer fc=wsManager.getFileById(wc.getStorageID()); String completeFilename=Files.fixFilename(fc.get().getName()); String filename=completeFilename.contains(".")?completeFilename.substring(0, completeFilename.lastIndexOf(".")):completeFilename; Destination destination=new Destination(completeFilename); destination.setCreateSubfolders(true); destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); destination.setPersistenceId("geoserver"); destination.setSubFolder("GNA/"+projectId+"/"+ currentElement.getMongo_id()+"/"+filename); log.debug("Sending "+wc+" to "+destination); TransferResult result=dtGeoServer.httpSource(fc.getPublicLink(), destination); log.debug("Transferred "+result); content.getFileNames().add(completeFilename); content.setGeoserverPath(result.getRemotePath().substring(0,result.getRemotePath().lastIndexOf("/"))); } }catch(Throwable t) { log.warn("Unable to transfer Persisted content"+c,t); } }); if(content.getFileNames().isEmpty()) throw new SDIInteractionException("No Persisted content found in "+currentElement); String completeFileName=content.getFileNames().get(0); String filename=completeFileName.contains(".")?completeFileName.substring(0, completeFileName.lastIndexOf(".")):completeFileName; String remoteFolder=content.getGeoserverPath(); String toSetLayerName=filename; int count=0; GeoServerRESTReader gsReader=currentGeoserver.getReader(); while(gsReader.getLayer(workspace,toSetLayerName)!=null){ count++; toSetLayerName=filename+"_"+count; log.debug("layer for "+filename+" already existing, trying "+toSetLayerName); } String storeName=toSetLayerName+"_store"; content.setStore(storeName); content.setFeatureType(toSetLayerName); GeoServerRESTPublisher publisher=currentGeoserver.getPublisher(); log.debug("Trying to create remote workspace : "+workspace); createWorkspace(workspace); log.debug("Publishing remote folder "+remoteFolder); URL directoryPath=new URL("file:"+remoteFolder+"/"+filename+".shp"); //TODO Evaluate SRS boolean published=publisher.publishShp( workspace, storeName, null, toSetLayerName, // UploadMethod.FILE, // neeeds zip UploadMethod.EXTERNAL, // needs shp directoryPath.toURI(), EPSG_4326, //SRS ""); // default style if(!published) { throw new SDIInteractionException("Unable to publish layer "+toSetLayerName+" under "+workspace+". Unknown Geoserver fault."); } currentElement.setLayerName(toSetLayerName); RESTLayer l=gsReader.getLayer(workspace, toSetLayerName); RESTFeatureType f= gsReader.getFeatureType(l); /*http://geoserver1.dev.d4science.org/geoserver/gna_conc_18/wms? service=WMS&version=1.1.0&request=GetMap&layers=gna_conc_18:pos& styles=&bbox=8.62091913167495,40.62975046683799,8.621178639172953,40.630257904721645& width=392&height=768&srs=EPSG:4326&format=application/openlayers */ currentElement.setWmsLink( String.format("https://%1$s/geoserver/%2$s/wms?" +"service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&" + "styles=&bbox=%4$f,%5$f,%6$f,%7$f&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d", geoserverHostName, workspace, toSetLayerName, f.getMinX(), f.getMinY(), f.getMaxX(), f.getMaxY(), EPSG_4326, 400, 400)); currentElement.setWorkspace(workspace); currentElement.setBbox(new BBOX(f.getMaxY(), f.getMaxX(), f.getMinY(), f.getMinX())); // TODO Metadata return content; // } catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException // | InvalidDestinationException | DestinationNotSetException e) { // throw new SDIInteractionException("Unable to transfer fileSet for content "+currentElement,e); } catch (SDIInteractionException e) { throw e; } catch (Throwable t) { throw new SDIInteractionException("Unexpected internal fault while interacting with SDI.",t); } } private String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException { String storeName=encoder.getName(); try { log.debug("Looking for datastore "+storeName+" under "+workspace); if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null) if(!currentGeoserver.getDataStoreManager().create(workspace, encoder)) throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace); log.debug("Store "+storeName+" exists under "+workspace); return storeName; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create store "+storeName,e); } } private String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{ String connectionUrl=connection.getUrl(); Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl); if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection); Matcher port = PORT_PATTERN.matcher(connectionUrl); if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection); Matcher db = DB_NAME_PATTERN.matcher(connectionUrl); if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection); GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); encoder.setHost(hostname.group()); encoder.setPort(Integer.parseInt(port.group())); encoder.setDatabase(db.group()); encoder.setSchema("public"); encoder.setUser(connection.getUser()); encoder.setPassword(connection.getPwd()); encoder.setLooseBBox(true); encoder.setDatabaseType("postgis"); encoder.setEnabled(true); encoder.setFetchSize(1000); encoder.setValidateConnections(true); return createStore(encoder,workspace); } private String createStoreFromJNDIDB(String workspace,String storeName) throws SDIInteractionException { //SET BY PROVISIONING GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); encoder.setJndiReferenceName("java:comp/env/jdbc/postgres"); encoder.setLooseBBox(true); encoder.setDatabaseType("postgis"); encoder.setEnabled(true); encoder.setFetchSize(1000); encoder.setValidateConnections(true); return createStore(encoder,workspace); } private String publishStyle(File sldFile,String name) throws SDIInteractionException { try { if(!currentGeoserver.getReader().existsStyle(name)) { log.debug("Registering style "+name); if(!currentGeoserver.getPublisher().publishStyle(sldFile, name)) throw new SDIInteractionException("Unable to register style "+name); }else log.debug("Style "+name+" already existing"); return name; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create style "+name,e); } } public String configureCentroidLayer(String name,String workspace,String storeName,PostgisTable table, DatabaseConnection connection) throws SDIInteractionException { GSFeatureTypeEncoder fte=new GSFeatureTypeEncoder(); fte.setAbstract("Centroid layer for "+name); fte.setEnabled(true); fte.setNativeCRS(WGS84_FULL); fte.setTitle(name); fte.setName(name); // GeoServer loads all fields // fte.setAttribute(attrs); fte.setLatLonBoundingBox(-180.0, -90.0, 180.0, 90.0, WGS84_FULL); String style="clustered_centroids"; GSLayerEncoder layerEncoder=new GSLayerEncoder(); layerEncoder.setDefaultStyle(style); layerEncoder.setEnabled(true); layerEncoder.setQueryable(true); try { //Checking workspace createWorkspace(workspace); //Checking store createStoreFromPostgisDB(workspace, storeName,connection); //Checking layer publishStyle(Files.getFileFromResources("styles/clustered_points.sld"),style); if(currentGeoserver.getReader().getLayer(workspace, name)==null) if(!currentGeoserver.getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder)) throw new SDIInteractionException("Unable to create layer "+name); log.debug("layer "+name+" already exists"); return name; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create layer "+name,e); } } public void deleteContent(GeoServerContent toDelete) throws IllegalArgumentException, MalformedURLException, RemoteServiceException { log.debug("Deleting geoserver layer "+toDelete); //delete layer GeoServerRESTPublisher publisher=currentGeoserver.getPublisher(); //delete store publisher.removeDatastore(toDelete.getWorkspace(), toDelete.getStore(), true); //delete WS if empty GeoServerRESTReader reader=currentGeoserver.getReader(); if(reader.getDatastores(toDelete.getWorkspace()).isEmpty()) { log.debug("Deleting emtpy workspace "+toDelete.getWorkspace()); publisher.removeWorkspace(toDelete.getWorkspace(), true); } //delete file dtGeoServer.getWebClient().delete(toDelete.getGeoserverPath()); } }