You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
435 lines
16 KiB
Java
435 lines
16 KiB
Java
package org.gcube.application.geoportal.service.engine.materialization;
|
|
|
|
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher;
|
|
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod;
|
|
import it.geosolutions.geoserver.rest.GeoServerRESTReader;
|
|
import it.geosolutions.geoserver.rest.decoder.RESTFeatureType;
|
|
import it.geosolutions.geoserver.rest.decoder.RESTLayer;
|
|
import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder;
|
|
import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder;
|
|
import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder;
|
|
import lombok.Getter;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.gcube.application.geoportal.common.model.legacy.*;
|
|
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
|
|
import org.gcube.application.geoportal.common.utils.Files;
|
|
import org.gcube.application.geoportal.service.engine.WorkspaceManager;
|
|
import org.gcube.application.geoportal.service.model.internal.db.PostgisTable;
|
|
import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException;
|
|
import org.gcube.common.storagehub.client.dsl.FileContainer;
|
|
import org.gcube.data.transfer.library.DataTransferClient;
|
|
import org.gcube.data.transfer.library.TransferResult;
|
|
import org.gcube.data.transfer.library.faults.RemoteServiceException;
|
|
import org.gcube.data.transfer.model.Destination;
|
|
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
|
import org.gcube.data.transfer.model.RemoteFileDescriptor;
|
|
import org.gcube.spatial.data.gis.GISInterface;
|
|
import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor;
|
|
|
|
import java.io.File;
|
|
import java.net.MalformedURLException;
|
|
import java.net.URL;
|
|
import java.util.function.Consumer;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
|
|
@Slf4j
|
|
public class SDIManager {
|
|
|
|
static private final String EPSG_4326="EPSG:4326";
|
|
static private final String WGS84_FULL="GEOGCS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", SPHEROID[\"WGS 84\", 6378137.0, 298.257223563, AUTHORITY[\"EPSG\",\"7030\"]],"+
|
|
"AUTHORITY[\"EPSG\",\"6326\"]], PRIMEM[\"Greenwich\", 0.0, AUTHORITY[\"EPSG\",\"8901\"]], UNIT[\"degree\", 0.017453292519943295],"+
|
|
"AXIS[\"Geodetic longitude\", EAST], AXIS[\"Geodetic latitude\", NORTH], AUTHORITY[\"EPSG\",\"4326\"]]";
|
|
|
|
|
|
public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*");
|
|
public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+");
|
|
public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)");
|
|
|
|
|
|
private final GISInterface gis;
|
|
@Getter
|
|
private final DataTransferClient dtGeoServer;
|
|
private final String geoserverHostName;
|
|
|
|
private final AbstractGeoServerDescriptor currentGeoserver;
|
|
|
|
|
|
public SDIManager() throws SDIInteractionException {
|
|
try{
|
|
log.debug("Initializing GIS Interface..");
|
|
gis=GISInterface.get();
|
|
currentGeoserver=gis.getCurrentGeoServer();
|
|
if(currentGeoserver==null)
|
|
throw new Exception("Unable to contact data transfer for geoserver ");
|
|
|
|
log.debug("Found geoserver descriptor "+currentGeoserver);
|
|
geoserverHostName=new URL(currentGeoserver.getUrl()).getHost();
|
|
|
|
log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName);
|
|
dtGeoServer=DataTransferClient.getInstanceByEndpoint("http://"+geoserverHostName);
|
|
if(!currentGeoserver.getReader().existGeoserver())
|
|
throw new Exception("Geoserver not reachable");
|
|
}catch(Exception e) {
|
|
throw new SDIInteractionException("Unable to initialize SDI Manager",e);
|
|
}
|
|
}
|
|
|
|
|
|
public RemoteFileDescriptor getGeoServerRemoteFolder() throws RemoteServiceException {
|
|
return dtGeoServer.getWebClient().getInfo("geoserver/GNA");
|
|
}
|
|
|
|
public String createWorkspace(String toCreate) throws SDIInteractionException {
|
|
try {
|
|
if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) {
|
|
log.debug("Creating workspace : "+toCreate);
|
|
if(!currentGeoserver.getPublisher().createWorkspace(toCreate))
|
|
throw new SDIInteractionException("Unable to create workspace "+toCreate);
|
|
}else log.debug("Workspace "+toCreate+" exists.");
|
|
return toCreate;
|
|
} catch (IllegalArgumentException | MalformedURLException e) {
|
|
throw new SDIInteractionException("Unable to create workspace "+toCreate,e);
|
|
}
|
|
}
|
|
|
|
|
|
// GEOSERVER-PERSISTENCE-ID / GNA / PROJECT-ID/ LAYER-ID /FILENAME(no extension)/...
|
|
|
|
public GeoServerContent pushShapeLayerFileSet(SDILayerDescriptor currentElement,String workspace, String projectId) throws SDIInteractionException{
|
|
try {
|
|
// String remoteFolder=null;
|
|
// String fileName=null;
|
|
|
|
log.debug("Publishing "+currentElement+" files to geoserver @ "+geoserverHostName);
|
|
|
|
if(currentElement.getActualContent()==null||currentElement.getActualContent().isEmpty())
|
|
throw new SDIInteractionException("Nothing to publish");
|
|
|
|
GeoServerContent content=new GeoServerContent();
|
|
content.setGeoserverHostName(geoserverHostName);
|
|
content.setWorkspace(workspace);
|
|
WorkspaceManager wsManager=new WorkspaceManager();
|
|
|
|
|
|
// ******** IDENTIFY LAYER NAME correct layer name
|
|
// Must be unique under same WS
|
|
// equal to shp base name
|
|
|
|
String baseName= "";
|
|
|
|
// Chose layer name first identifying filename
|
|
for(PersistedContent p:currentElement.getActualContent()){
|
|
if(p instanceof WorkspaceContent) {
|
|
WorkspaceContent w= (WorkspaceContent) p;
|
|
if(w.getName().endsWith(".shp")) {
|
|
log.debug("SHP is {}",w.getName());
|
|
baseName=Files.fixFilename(w.getName().substring(0,w.getName().lastIndexOf('.')));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
String toSetLayerName=baseName;
|
|
|
|
//Check if layer already exists
|
|
int count=0;
|
|
GeoServerRESTReader gsReader=currentGeoserver.getReader();
|
|
while(gsReader.getLayer(workspace,toSetLayerName)!=null){
|
|
count++;
|
|
toSetLayerName=baseName+"_"+count;
|
|
log.debug("layer for "+baseName+" already existing, trying "+toSetLayerName);
|
|
}
|
|
|
|
|
|
String folderRelativePath="GNA/" + projectId + "/" +
|
|
currentElement.getMongo_id() + "/" + toSetLayerName;
|
|
|
|
|
|
for (PersistedContent c : currentElement.getActualContent()) {
|
|
if (c instanceof WorkspaceContent) {
|
|
WorkspaceContent wc = (WorkspaceContent) c;
|
|
FileContainer fc = wsManager.getFileById(wc.getStorageID());
|
|
|
|
String completeFilename = Files.fixFilename(fc.get().getName());
|
|
|
|
completeFilename=completeFilename.replaceAll(baseName, toSetLayerName);
|
|
|
|
|
|
Destination destination = new Destination(completeFilename);
|
|
destination.setCreateSubfolders(true);
|
|
destination.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
|
destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
|
|
|
destination.setPersistenceId("geoserver");
|
|
destination.setSubFolder(folderRelativePath);
|
|
|
|
log.debug("Sending " + wc + " to " + destination);
|
|
TransferResult result = SDIManager.this.getDtGeoServer().httpSource(fc.getPublicLink(), destination);
|
|
log.debug("Transferred " + result);
|
|
|
|
|
|
content.getFileNames().add(completeFilename);
|
|
|
|
content.setGeoserverPath(result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/")));
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
// String completeFileName=content.getFileNames().get(0);
|
|
// String filename=completeFileName.contains(".")?completeFileName.substring(0, completeFileName.lastIndexOf(".")):completeFileName;
|
|
//
|
|
// String remoteFolder=content.getGeoserverPath();
|
|
|
|
|
|
|
|
|
|
|
|
String storeName=toSetLayerName+"_store";
|
|
content.setStore(storeName);
|
|
content.setFeatureType(toSetLayerName);
|
|
|
|
GeoServerRESTPublisher publisher=currentGeoserver.getPublisher();
|
|
log.debug("Trying to create remote workspace : "+workspace);
|
|
createWorkspace(workspace);
|
|
|
|
String folderAbsolutePath=content.getGeoserverPath();
|
|
log.debug("Publishing remote folder "+folderAbsolutePath);
|
|
|
|
URL directoryPath=new URL("file:"+folderAbsolutePath+"/"+toSetLayerName+".shp");
|
|
|
|
|
|
//TODO Evaluate SRS
|
|
|
|
boolean published=publisher.publishShp(
|
|
workspace,
|
|
storeName,
|
|
null,
|
|
toSetLayerName,
|
|
// UploadMethod.FILE, // neeeds zip
|
|
UploadMethod.EXTERNAL, // needs shp
|
|
directoryPath.toURI(),
|
|
EPSG_4326, //SRS
|
|
""); // default style
|
|
|
|
if(!published) {
|
|
throw new SDIInteractionException("Unable to publish layer "+toSetLayerName+" under "+workspace+". Unknown Geoserver fault.");
|
|
}
|
|
|
|
currentElement.setLayerName(toSetLayerName);
|
|
|
|
RESTLayer l=gsReader.getLayer(workspace, toSetLayerName);
|
|
RESTFeatureType f= gsReader.getFeatureType(l);
|
|
/*http://geoserver1.dev.d4science.org/geoserver/gna_conc_18/wms?
|
|
service=WMS&version=1.1.0&request=GetMap&layers=gna_conc_18:pos&
|
|
styles=&bbox=8.62091913167495,40.62975046683799,8.621178639172953,40.630257904721645&
|
|
width=392&height=768&srs=EPSG:4326&format=application/openlayers */
|
|
|
|
|
|
currentElement.setWmsLink(
|
|
String.format("https://%1$s/geoserver/%2$s/wms?"
|
|
+"service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&"
|
|
+ "styles=&bbox=%4$f,%5$f,%6$f,%7$f&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d",
|
|
geoserverHostName,
|
|
workspace,
|
|
toSetLayerName,
|
|
f.getMinX(),
|
|
f.getMinY(),
|
|
f.getMaxX(),
|
|
f.getMaxY(),
|
|
EPSG_4326,
|
|
400,
|
|
400));
|
|
|
|
currentElement.setWorkspace(workspace);
|
|
currentElement.setBbox(new BBOX(f.getMaxY(), f.getMaxX(), f.getMinY(), f.getMinX()));
|
|
|
|
|
|
// TODO Metadata
|
|
return content;
|
|
// } catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException
|
|
// | InvalidDestinationException | DestinationNotSetException e) {
|
|
// throw new SDIInteractionException("Unable to transfer fileSet for content "+currentElement,e);
|
|
} catch (SDIInteractionException e) {
|
|
throw e;
|
|
} catch (Throwable t) {
|
|
throw new SDIInteractionException("Unexpected internal fault while interacting with SDI.",t);
|
|
}
|
|
}
|
|
|
|
private String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException {
|
|
String storeName=encoder.getName();
|
|
try {
|
|
log.debug("Looking for datastore "+storeName+" under "+workspace);
|
|
|
|
if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null)
|
|
|
|
if(!currentGeoserver.getDataStoreManager().create(workspace, encoder))
|
|
throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace);
|
|
log.debug("Store "+storeName+" exists under "+workspace);
|
|
return storeName;
|
|
} catch (IllegalArgumentException | MalformedURLException e) {
|
|
throw new SDIInteractionException("Unable to create store "+storeName,e);
|
|
}
|
|
}
|
|
|
|
private String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{
|
|
String connectionUrl=connection.getUrl();
|
|
|
|
Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl);
|
|
if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection);
|
|
|
|
Matcher port = PORT_PATTERN.matcher(connectionUrl);
|
|
if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection);
|
|
|
|
Matcher db = DB_NAME_PATTERN.matcher(connectionUrl);
|
|
if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection);
|
|
|
|
|
|
GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName);
|
|
encoder.setHost(hostname.group());
|
|
encoder.setPort(Integer.parseInt(port.group()));
|
|
encoder.setDatabase(db.group());
|
|
encoder.setSchema("public");
|
|
encoder.setUser(connection.getUser());
|
|
encoder.setPassword(connection.getPwd());
|
|
encoder.setLooseBBox(true);
|
|
encoder.setDatabaseType("postgis");
|
|
encoder.setEnabled(true);
|
|
encoder.setFetchSize(1000);
|
|
encoder.setValidateConnections(true);
|
|
|
|
return createStore(encoder,workspace);
|
|
}
|
|
private String createStoreFromJNDIDB(String workspace,String storeName) throws SDIInteractionException {
|
|
//SET BY PROVISIONING
|
|
GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName);
|
|
encoder.setJndiReferenceName("java:comp/env/jdbc/postgres");
|
|
encoder.setLooseBBox(true);
|
|
encoder.setDatabaseType("postgis");
|
|
encoder.setEnabled(true);
|
|
encoder.setFetchSize(1000);
|
|
encoder.setValidateConnections(true);
|
|
return createStore(encoder,workspace);
|
|
}
|
|
|
|
private String publishStyle(File sldFile,String name) throws SDIInteractionException {
|
|
try {
|
|
if(!currentGeoserver.getReader().existsStyle(name)) {
|
|
log.debug("Registering style "+name);
|
|
if(!currentGeoserver.getPublisher().publishStyle(sldFile, name))
|
|
throw new SDIInteractionException("Unable to register style "+name);
|
|
}else log.debug("Style "+name+" already existing");
|
|
return name;
|
|
} catch (IllegalArgumentException | MalformedURLException e) {
|
|
throw new SDIInteractionException("Unable to create style "+name,e);
|
|
}
|
|
|
|
}
|
|
|
|
public String configureCentroidLayer(String name, String workspace, String storeName, PostgisTable table, DatabaseConnection connection) throws SDIInteractionException {
|
|
|
|
GSFeatureTypeEncoder fte=new GSFeatureTypeEncoder();
|
|
fte.setAbstract("Centroid layer for "+name);
|
|
fte.setEnabled(true);
|
|
fte.setNativeCRS(WGS84_FULL);
|
|
fte.setTitle(name);
|
|
fte.setName(name);
|
|
|
|
|
|
// GeoServer loads all fields
|
|
// fte.setAttribute(attrs);
|
|
|
|
|
|
fte.setLatLonBoundingBox(-180.0, -90.0, 180.0, 90.0, WGS84_FULL);
|
|
|
|
String style="clustered_centroids";
|
|
|
|
GSLayerEncoder layerEncoder=new GSLayerEncoder();
|
|
layerEncoder.setDefaultStyle(style);
|
|
layerEncoder.setEnabled(true);
|
|
layerEncoder.setQueryable(true);
|
|
try {
|
|
//Checking workspace
|
|
createWorkspace(workspace);
|
|
//Checking store
|
|
createStoreFromPostgisDB(workspace, storeName,connection);
|
|
//Checking layer
|
|
publishStyle(Files.getFileFromResources("styles/clustered_points.sld"),style);
|
|
|
|
log.info("Creating layer in {} : {} with FTE {} , LE {}",workspace,storeName,fte,layerEncoder);
|
|
if(currentGeoserver.getReader().getLayer(workspace, name)==null)
|
|
if(!currentGeoserver.getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder))
|
|
throw new SDIInteractionException("Unable to create layer "+name);
|
|
log.debug("layer "+name+" already exists");
|
|
|
|
|
|
String link=String.format("https://%1$s/geoserver/%2$s/wms?"
|
|
+"service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&"
|
|
+ "styles=&bbox=%4$s,%5$s,%6$s,%7$s&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d",
|
|
geoserverHostName,
|
|
workspace,
|
|
name,
|
|
"-1563071.166172796",
|
|
"4789738.204048398",
|
|
"4334926.486925308",
|
|
"5828118.072551585",
|
|
EPSG_4326,
|
|
400,
|
|
400);
|
|
|
|
return name;
|
|
} catch (IllegalArgumentException | MalformedURLException e) {
|
|
throw new SDIInteractionException("Unable to create layer "+name,e);
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
public void deleteContent(GeoServerContent toDelete) throws IllegalArgumentException, MalformedURLException, RemoteServiceException {
|
|
log.info("Deleting geoserver layer "+toDelete);
|
|
|
|
String geoserverHostName=toDelete.getGeoserverHostName();
|
|
log.debug("Looking for geoserver {}",geoserverHostName);
|
|
|
|
AbstractGeoServerDescriptor geoServerDescriptor=null;
|
|
|
|
for(AbstractGeoServerDescriptor gs :gis.getCurrentCacheElements(false)){
|
|
log.debug("Checking gs {}",gs);
|
|
if(new URL(gs.getUrl()).getHost().equals(geoserverHostName))
|
|
geoServerDescriptor=gs;
|
|
}
|
|
if(geoServerDescriptor == null) throw new IllegalArgumentException("Unable to find geoserver "+geoserverHostName);
|
|
|
|
GeoServerRESTPublisher publisher=geoServerDescriptor.getPublisher();
|
|
|
|
//delete layer
|
|
//delete store
|
|
log.debug("Removing DS {} : {} ",toDelete.getWorkspace(),toDelete.getStore());
|
|
publisher.removeDatastore(toDelete.getWorkspace(), toDelete.getStore(), true);
|
|
|
|
//delete WS if empty
|
|
GeoServerRESTReader reader=geoServerDescriptor.getReader();
|
|
log.debug("Checking if WS {} is empty",toDelete.getWorkspace());
|
|
if(reader.getDatastores(toDelete.getWorkspace()).isEmpty()) {
|
|
log.debug("Deleting emtpy workspace "+toDelete.getWorkspace());
|
|
publisher.removeWorkspace(toDelete.getWorkspace(), true);
|
|
}
|
|
//delete file
|
|
|
|
|
|
// TODO REMOVE HARDCODED PATCH
|
|
String path=toDelete.getGeoserverPath().replace("/srv/geoserver_data","geoserver");
|
|
log.info("Deleting files at {} [{}]",path,toDelete.getGeoserverPath());
|
|
// path=toDelete.getGeoserverPath();
|
|
dtGeoServer.getWebClient().delete(path);
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|