package org.gcube.application.geoportal.service.engine.materialization; import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod; import it.geosolutions.geoserver.rest.GeoServerRESTReader; import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; import it.geosolutions.geoserver.rest.decoder.RESTLayer; import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder; import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder; import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.application.geoportal.service.model.internal.db.PostgisTable; import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; import org.gcube.common.storagehub.client.dsl.FileContainer; import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.library.faults.RemoteServiceException; import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.spatial.data.gis.GISInterface; import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; import java.io.File; import java.net.MalformedURLException; import java.net.URL; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @Slf4j public class SDIManager { static private final String EPSG_4326="EPSG:4326"; static private final String WGS84_FULL="GEOGCS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", SPHEROID[\"WGS 84\", 6378137.0, 298.257223563, AUTHORITY[\"EPSG\",\"7030\"]],"+ "AUTHORITY[\"EPSG\",\"6326\"]], PRIMEM[\"Greenwich\", 0.0, AUTHORITY[\"EPSG\",\"8901\"]], UNIT[\"degree\", 0.017453292519943295],"+ "AXIS[\"Geodetic longitude\", EAST], AXIS[\"Geodetic latitude\", NORTH], AUTHORITY[\"EPSG\",\"4326\"]]"; public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+"); public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); private final GISInterface gis; @Getter private final DataTransferClient dtGeoServer; private final String geoserverHostName; private final AbstractGeoServerDescriptor currentGeoserver; public SDIManager() throws SDIInteractionException { try{ log.debug("Initializing GIS Interface.."); gis=GISInterface.get(); currentGeoserver=gis.getCurrentGeoServer(); if(currentGeoserver==null) throw new Exception("Unable to contact data transfer for geoserver "); log.debug("Found geoserver descriptor "+currentGeoserver); geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); dtGeoServer=DataTransferClient.getInstanceByEndpoint("http://"+geoserverHostName); if(!currentGeoserver.getReader().existGeoserver()) throw new Exception("Geoserver not reachable"); }catch(Exception e) { throw new SDIInteractionException("Unable to initialize SDI Manager",e); } } public RemoteFileDescriptor getGeoServerRemoteFolder() throws RemoteServiceException { return dtGeoServer.getWebClient().getInfo("geoserver/GNA"); } public String createWorkspace(String toCreate) throws SDIInteractionException { try { if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { log.debug("Creating workspace : "+toCreate); if(!currentGeoserver.getPublisher().createWorkspace(toCreate)) throw new SDIInteractionException("Unable to create workspace "+toCreate); }else log.debug("Workspace "+toCreate+" exists."); return toCreate; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create workspace "+toCreate,e); } } // GEOSERVER-PERSISTENCE-ID / GNA / PROJECT-ID/ LAYER-ID /FILENAME(no extension)/... public GeoServerContent pushShapeLayerFileSet(SDILayerDescriptor currentElement,String workspace, String projectId) throws SDIInteractionException{ try { // String remoteFolder=null; // String fileName=null; log.debug("Publishing "+currentElement+" files to geoserver @ "+geoserverHostName); if(currentElement.getActualContent()==null||currentElement.getActualContent().isEmpty()) throw new SDIInteractionException("Nothing to publish"); GeoServerContent content=new GeoServerContent(); content.setGeoserverHostName(geoserverHostName); content.setWorkspace(workspace); WorkspaceManager wsManager=new WorkspaceManager(); // ******** IDENTIFY LAYER NAME correct layer name // Must be unique under same WS // equal to shp base name String baseName= ""; // Chose layer name first identifying filename for(PersistedContent p:currentElement.getActualContent()){ if(p instanceof WorkspaceContent) { WorkspaceContent w= (WorkspaceContent) p; if(w.getName().endsWith(".shp")) { log.debug("SHP is {}",w.getName()); baseName=Files.fixFilename(w.getName().substring(0,w.getName().lastIndexOf('.'))); break; } } } String toSetLayerName=baseName; //Check if layer already exists int count=0; GeoServerRESTReader gsReader=currentGeoserver.getReader(); while(gsReader.getLayer(workspace,toSetLayerName)!=null){ count++; toSetLayerName=baseName+"_"+count; log.debug("layer for "+baseName+" already existing, trying "+toSetLayerName); } String folderRelativePath="GNA/" + projectId + "/" + currentElement.getMongo_id() + "/" + toSetLayerName; for (PersistedContent c : currentElement.getActualContent()) { if (c instanceof WorkspaceContent) { WorkspaceContent wc = (WorkspaceContent) c; FileContainer fc = wsManager.getFileById(wc.getStorageID()); String completeFilename = Files.fixFilename(fc.get().getName()); completeFilename=completeFilename.replaceAll(baseName, toSetLayerName); Destination destination = new Destination(completeFilename); destination.setCreateSubfolders(true); destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); destination.setPersistenceId("geoserver"); destination.setSubFolder(folderRelativePath); log.debug("Sending " + wc + " to " + destination); TransferResult result = SDIManager.this.getDtGeoServer().httpSource(fc.getPublicLink(), destination); log.debug("Transferred " + result); content.getFileNames().add(completeFilename); content.setGeoserverPath(result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/"))); } } // String completeFileName=content.getFileNames().get(0); // String filename=completeFileName.contains(".")?completeFileName.substring(0, completeFileName.lastIndexOf(".")):completeFileName; // // String remoteFolder=content.getGeoserverPath(); String storeName=toSetLayerName+"_store"; content.setStore(storeName); content.setFeatureType(toSetLayerName); GeoServerRESTPublisher publisher=currentGeoserver.getPublisher(); log.debug("Trying to create remote workspace : "+workspace); createWorkspace(workspace); String folderAbsolutePath=content.getGeoserverPath(); log.debug("Publishing remote folder "+folderAbsolutePath); URL directoryPath=new URL("file:"+folderAbsolutePath+"/"+toSetLayerName+".shp"); //TODO Evaluate SRS boolean published=publisher.publishShp( workspace, storeName, null, toSetLayerName, // UploadMethod.FILE, // neeeds zip UploadMethod.EXTERNAL, // needs shp directoryPath.toURI(), EPSG_4326, //SRS ""); // default style if(!published) { throw new SDIInteractionException("Unable to publish layer "+toSetLayerName+" under "+workspace+". Unknown Geoserver fault."); } currentElement.setLayerName(toSetLayerName); RESTLayer l=gsReader.getLayer(workspace, toSetLayerName); RESTFeatureType f= gsReader.getFeatureType(l); /*http://geoserver1.dev.d4science.org/geoserver/gna_conc_18/wms? service=WMS&version=1.1.0&request=GetMap&layers=gna_conc_18:pos& styles=&bbox=8.62091913167495,40.62975046683799,8.621178639172953,40.630257904721645& width=392&height=768&srs=EPSG:4326&format=application/openlayers */ currentElement.setWmsLink( String.format("https://%1$s/geoserver/%2$s/wms?" +"service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&" + "styles=&bbox=%4$f,%5$f,%6$f,%7$f&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d", geoserverHostName, workspace, toSetLayerName, f.getMinX(), f.getMinY(), f.getMaxX(), f.getMaxY(), EPSG_4326, 400, 400)); currentElement.setWorkspace(workspace); currentElement.setBbox(new BBOX(f.getMaxY(), f.getMaxX(), f.getMinY(), f.getMinX())); // TODO Metadata return content; // } catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException // | InvalidDestinationException | DestinationNotSetException e) { // throw new SDIInteractionException("Unable to transfer fileSet for content "+currentElement,e); } catch (SDIInteractionException e) { throw e; } catch (Throwable t) { throw new SDIInteractionException("Unexpected internal fault while interacting with SDI.",t); } } private String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException { String storeName=encoder.getName(); try { log.debug("Looking for datastore "+storeName+" under "+workspace); if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null) if(!currentGeoserver.getDataStoreManager().create(workspace, encoder)) throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace); log.debug("Store "+storeName+" exists under "+workspace); return storeName; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create store "+storeName,e); } } private String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{ String connectionUrl=connection.getUrl(); Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl); if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection); Matcher port = PORT_PATTERN.matcher(connectionUrl); if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection); Matcher db = DB_NAME_PATTERN.matcher(connectionUrl); if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection); GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); encoder.setHost(hostname.group()); encoder.setPort(Integer.parseInt(port.group())); encoder.setDatabase(db.group()); encoder.setSchema("public"); encoder.setUser(connection.getUser()); encoder.setPassword(connection.getPwd()); encoder.setLooseBBox(true); encoder.setDatabaseType("postgis"); encoder.setEnabled(true); encoder.setFetchSize(1000); encoder.setValidateConnections(true); return createStore(encoder,workspace); } private String createStoreFromJNDIDB(String workspace,String storeName) throws SDIInteractionException { //SET BY PROVISIONING GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); encoder.setJndiReferenceName("java:comp/env/jdbc/postgres"); encoder.setLooseBBox(true); encoder.setDatabaseType("postgis"); encoder.setEnabled(true); encoder.setFetchSize(1000); encoder.setValidateConnections(true); return createStore(encoder,workspace); } private String publishStyle(File sldFile,String name) throws SDIInteractionException { try { if(!currentGeoserver.getReader().existsStyle(name)) { log.debug("Registering style "+name); if(!currentGeoserver.getPublisher().publishStyle(sldFile, name)) throw new SDIInteractionException("Unable to register style "+name); }else log.debug("Style "+name+" already existing"); return name; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create style "+name,e); } } public String configureCentroidLayer(String name, String workspace, String storeName, PostgisTable table, DatabaseConnection connection) throws SDIInteractionException { GSFeatureTypeEncoder fte=new GSFeatureTypeEncoder(); fte.setAbstract("Centroid layer for "+name); fte.setEnabled(true); fte.setNativeCRS(WGS84_FULL); fte.setTitle(name); fte.setName(name); // GeoServer loads all fields // fte.setAttribute(attrs); fte.setLatLonBoundingBox(-180.0, -90.0, 180.0, 90.0, WGS84_FULL); String style="clustered_centroids"; GSLayerEncoder layerEncoder=new GSLayerEncoder(); layerEncoder.setDefaultStyle(style); layerEncoder.setEnabled(true); layerEncoder.setQueryable(true); try { //Checking workspace createWorkspace(workspace); //Checking store createStoreFromPostgisDB(workspace, storeName,connection); //Checking layer publishStyle(Files.getFileFromResources("styles/clustered_points.sld"),style); log.info("Creating layer in {} : {} with FTE {} , LE {}",workspace,storeName,fte,layerEncoder); if(currentGeoserver.getReader().getLayer(workspace, name)==null) if(!currentGeoserver.getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder)) throw new SDIInteractionException("Unable to create layer "+name); log.debug("layer "+name+" already exists"); String link=String.format("https://%1$s/geoserver/%2$s/wms?" +"service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&" + "styles=&bbox=%4$s,%5$s,%6$s,%7$s&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d", geoserverHostName, workspace, name, "-1563071.166172796", "4789738.204048398", "4334926.486925308", "5828118.072551585", EPSG_4326, 400, 400); return name; } catch (IllegalArgumentException | MalformedURLException e) { throw new SDIInteractionException("Unable to create layer "+name,e); } } public void deleteContent(GeoServerContent toDelete) throws IllegalArgumentException, MalformedURLException, RemoteServiceException { log.info("Deleting geoserver layer "+toDelete); String geoserverHostName=toDelete.getGeoserverHostName(); log.debug("Looking for geoserver {}",geoserverHostName); AbstractGeoServerDescriptor geoServerDescriptor=null; for(AbstractGeoServerDescriptor gs :gis.getCurrentCacheElements(false)){ log.debug("Checking gs {}",gs); if(new URL(gs.getUrl()).getHost().equals(geoserverHostName)) geoServerDescriptor=gs; } if(geoServerDescriptor == null) throw new IllegalArgumentException("Unable to find geoserver "+geoserverHostName); GeoServerRESTPublisher publisher=geoServerDescriptor.getPublisher(); //delete layer //delete store log.debug("Removing DS {} : {} ",toDelete.getWorkspace(),toDelete.getStore()); publisher.removeDatastore(toDelete.getWorkspace(), toDelete.getStore(), true); //delete WS if empty GeoServerRESTReader reader=geoServerDescriptor.getReader(); log.debug("Checking if WS {} is empty",toDelete.getWorkspace()); if(reader.getDatastores(toDelete.getWorkspace()).isEmpty()) { log.debug("Deleting emtpy workspace "+toDelete.getWorkspace()); publisher.removeWorkspace(toDelete.getWorkspace(), true); } //delete file // TODO REMOVE HARDCODED PATCH String path=toDelete.getGeoserverPath().replace("/srv/geoserver_data","geoserver"); log.info("Deleting files at {} [{}]",path,toDelete.getGeoserverPath()); // path=toDelete.getGeoserverPath(); dtGeoServer.getWebClient().delete(path); } }