gCubeSystem
/
geoportal-service
Archived
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix centroids layer definition

Browse Source
master
Fabio Sinibaldi 3 years ago
parent e96d010858
commit 67fd71182c
6 changed files with 192 additions and 89 deletions
Show Stats Download Patch File Download Diff File

2
CHANGELOG.md
Unescape Escape View File

@ -4,6 +4,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
## [v1.0.5-SNAPSHOT] 2021-07-23
Upgrade to gcube-smartgears-bom 2.1.0
Fix register postgis table layer
Added PostgisIndexRecordManager
## [v1.0.4] 2020-11-11
Mongo integration with Concessione

181
src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java
Unescape Escape View File

@ -1,15 +1,23 @@
package org.gcube.application.geoportal.service.engine;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import org.gcube.application.geoportal.common.model.legacy.BBOX;
import org.gcube.application.geoportal.common.model.legacy.GeoServerContent;
import org.gcube.application.geoportal.common.model.legacy.PersistedContent;
import org.gcube.application.geoportal.common.model.legacy.SDILayerDescriptor;
import org.gcube.application.geoportal.common.model.legacy.WorkspaceContent;
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher;
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod;
import it.geosolutions.geoserver.rest.GeoServerRESTReader;
import it.geosolutions.geoserver.rest.decoder.RESTFeatureType;
import it.geosolutions.geoserver.rest.decoder.RESTLayer;
import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder;
import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.FeatureTypeAttribute;
import it.geosolutions.geoserver.rest.encoder.feature.GSAttributeEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.geoportal.common.model.legacy.*;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.model.db.DBConstants;
import org.gcube.application.geoportal.model.db.DatabaseConnection;
import org.gcube.application.geoportal.model.db.PostgisTable;
import org.gcube.application.geoportal.service.engine.mongo.PostgisIndex;
import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.data.transfer.library.DataTransferClient;
@ -21,43 +29,52 @@ import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.spatial.data.gis.GISInterface;
import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor;
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher;
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod;
import it.geosolutions.geoserver.rest.GeoServerRESTReader;
import it.geosolutions.geoserver.rest.decoder.RESTFeatureType;
import it.geosolutions.geoserver.rest.decoder.RESTLayer;
import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder;
import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
public class SDIManager {
static private String DEFAULT_CRS="EPSG:4326";
static private final String EPSG_4326="EPSG:4326";
static private final String WGS84_FULL="GEOGCS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", SPHEROID[\"WGS 84\", 6378137.0, 298.257223563, AUTHORITY[\"EPSG\",\"7030\"]],"+
"AUTHORITY[\"EPSG\",\"6326\"]], PRIMEM[\"Greenwich\", 0.0, AUTHORITY[\"EPSG\",\"8901\"]], UNIT[\"degree\", 0.017453292519943295],"+
"AXIS[\"Geodetic longitude\", EAST], AXIS[\"Geodetic latitude\", NORTH], AUTHORITY[\"EPSG\",\"4326\"]]";
private GISInterface gis;
public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*");
public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+");
public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)");
private final GISInterface gis;
@Getter
private DataTransferClient dtGeoServer;
private String geoserverHostName;
private final DataTransferClient dtGeoServer;
private final String geoserverHostName;
private final AbstractGeoServerDescriptor currentGeoserver;
public SDIManager() throws SDIInteractionException {
try{
log.debug("Initializing GIS Interface..");
gis=GISInterface.get();
AbstractGeoServerDescriptor geoserver=gis.getCurrentGeoServer();
if(geoserver==null)
currentGeoserver=gis.getCurrentGeoServer();
if(currentGeoserver==null)
throw new Exception("Unable to contact data transfer for geoserver ");
log.debug("Found geoserver descriptor "+geoserver);
geoserverHostName=new URL(gis.getCurrentGeoServer().getUrl()).getHost();
log.debug("Found geoserver descriptor "+currentGeoserver);
geoserverHostName=new URL(currentGeoserver.getUrl()).getHost();
log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName);
dtGeoServer=DataTransferClient.getInstanceByEndpoint("http://"+geoserverHostName);
if(!gis.getCurrentGeoServer().getReader().existGeoserver())
if(!currentGeoserver.getReader().existGeoserver())
throw new Exception("Geoserver not reachable");
}catch(Exception e) {
throw new SDIInteractionException("Unable to initialize SDI Manager",e);
@ -71,9 +88,9 @@ public class SDIManager {
public String createWorkspace(String toCreate) throws SDIInteractionException {
try {
if(!gis.getCurrentGeoServer().getReader().getWorkspaceNames().contains(toCreate)) {
if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) {
log.debug("Creating workspace : "+toCreate);
if(!gis.getCurrentGeoServer().getPublisher().createWorkspace(toCreate))
if(!currentGeoserver.getPublisher().createWorkspace(toCreate))
throw new SDIInteractionException("Unable to create workspace "+toCreate);
}else log.debug("Workspace "+toCreate+" exists.");
return toCreate;
@ -149,18 +166,19 @@ public class SDIManager {
String toSetLayerName=filename;
int count=0;
while(gis.getCurrentGeoServer().getReader().getLayer(workspace,toSetLayerName)!=null){
GeoServerRESTReader gsReader=currentGeoserver.getReader();
while(gsReader.getLayer(workspace,toSetLayerName)!=null){
count++;
toSetLayerName=filename+"_"+count;
log.debug("layer for "+filename+" already existing, trying "+toSetLayerName);
};
}
String storeName=toSetLayerName+"_store";
content.setStore(storeName);
content.setFeatureType(toSetLayerName);
GeoServerRESTPublisher publisher=gis.getCurrentGeoServer().getPublisher();
GeoServerRESTPublisher publisher=currentGeoserver.getPublisher();
log.debug("Trying to create remote workspace : "+workspace);
createWorkspace(workspace);
@ -179,7 +197,7 @@ public class SDIManager {
// UploadMethod.FILE, // neeeds zip
UploadMethod.EXTERNAL, // needs shp
directoryPath.toURI(),
DEFAULT_CRS, //SRS
EPSG_4326, //SRS
""); // default style
if(!published) {
@ -187,9 +205,9 @@ public class SDIManager {
}
currentElement.setLayerName(toSetLayerName);
GeoServerRESTReader reader=gis.getCurrentGeoServer().getReader();
RESTLayer l=reader.getLayer(workspace, toSetLayerName);
RESTFeatureType f= reader.getFeatureType(l);
RESTLayer l=gsReader.getLayer(workspace, toSetLayerName);
RESTFeatureType f= gsReader.getFeatureType(l);
/*http://geoserver1.dev.d4science.org/geoserver/gna_conc_18/wms?
service=WMS&version=1.1.0&request=GetMap&layers=gna_conc_18:pos&
styles=&bbox=8.62091913167495,40.62975046683799,8.621178639172953,40.630257904721645&
@ -207,7 +225,7 @@ public class SDIManager {
f.getMinY(),
f.getMaxX(),
f.getMaxY(),
DEFAULT_CRS,
EPSG_4326,
400,
400));
@ -227,35 +245,67 @@ public class SDIManager {
}
}
private String createStoreFromPostgisDB(String workspace,String storeName) throws SDIInteractionException {
//SET BY PROVISIONING
GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName);
encoder.setJndiReferenceName("java:comp/env/jdbc/postgres");
encoder.setLooseBBox(true);
encoder.setDatabaseType("postgis");
encoder.setEnabled(true);
encoder.setFetchSize(1000);
encoder.setValidateConnections(true);
private String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException {
String storeName=encoder.getName();
try {
log.debug("Looking for datastore "+storeName+" under "+workspace);
if(gis.getCurrentGeoServer().getReader().getDatastore(workspace,storeName)==null)
if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null)
if(!gis.getCurrentGeoServer().getDataStoreManager().create(workspace, encoder))
if(!currentGeoserver.getDataStoreManager().create(workspace, encoder))
throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace);
log.debug("Store "+storeName+" exists under "+workspace);
return storeName;
} catch (IllegalArgumentException | MalformedURLException e) {
throw new SDIInteractionException("Unable to create store "+storeName,e);
}
}
private String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{
String connectionUrl=connection.getUrl();
Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl);
if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection);
Matcher port = PORT_PATTERN.matcher(connectionUrl);
if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection);
Matcher db = DB_NAME_PATTERN.matcher(connectionUrl);
if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection);
GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName);
encoder.setHost(hostname.group());
encoder.setPort(Integer.parseInt(port.group()));
encoder.setDatabase(db.group());
encoder.setSchema("public");
encoder.setUser(connection.getUser());
encoder.setPassword(connection.getPwd());
encoder.setLooseBBox(true);
encoder.setDatabaseType("postgis");
encoder.setEnabled(true);
encoder.setFetchSize(1000);
encoder.setValidateConnections(true);
return createStore(encoder,workspace);
}
private String createStoreFromJNDIDB(String workspace,String storeName) throws SDIInteractionException {
//SET BY PROVISIONING
GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName);
encoder.setJndiReferenceName("java:comp/env/jdbc/postgres");
encoder.setLooseBBox(true);
encoder.setDatabaseType("postgis");
encoder.setEnabled(true);
encoder.setFetchSize(1000);
encoder.setValidateConnections(true);
return createStore(encoder,workspace);
}
private String publishStyle(File sldFile,String name) throws SDIInteractionException {
try {
if(!gis.getCurrentGeoServer().getReader().existsStyle(name)) {
if(!currentGeoserver.getReader().existsStyle(name)) {
log.debug("Registering style "+name);
if(!gis.getCurrentGeoServer().getPublisher().publishStyle(sldFile, name))
if(!currentGeoserver.getPublisher().publishStyle(sldFile, name))
throw new SDIInteractionException("Unable to register style "+name);
}else log.debug("Style "+name+" already existing");
return name;
@ -265,15 +315,21 @@ public class SDIManager {
}
public String configureCentroidLayer(String name,String workspace,String storeName) throws SDIInteractionException {
public String configureCentroidLayer(String name,String workspace,String storeName,PostgisTable table, DatabaseConnection connection) throws SDIInteractionException {
GSFeatureTypeEncoder fte=new GSFeatureTypeEncoder();
fte.setAbstract("Centroid layer for "+name);
fte.setEnabled(true);
fte.setNativeCRS(DEFAULT_CRS);
fte.setNativeCRS(WGS84_FULL);
fte.setTitle(name);
fte.setName(name);
// GeoServer loads all fields
// fte.setAttribute(attrs);
fte.setLatLonBoundingBox(-180.0, -90.0, 180.0, 90.0, WGS84_FULL);
String style="clustered_centroids";
@ -285,11 +341,12 @@ public class SDIManager {
//Checking workspace
createWorkspace(workspace);
//Checking store
createStoreFromPostgisDB(workspace, storeName);
//Checkig layer
createStoreFromPostgisDB(workspace, storeName,connection);
//Checking layer
publishStyle(Files.getFileFromResources("styles/clustered_points.sld"),style);
if(gis.getCurrentGeoServer().getReader().getLayer(workspace, name)==null)
if(!gis.getCurrentGeoServer().getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder))
if(currentGeoserver.getReader().getLayer(workspace, name)==null)
if(!currentGeoserver.getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder))
throw new SDIInteractionException("Unable to create layer "+name);
log.debug("layer "+name+" already exists");
return name;
@ -304,11 +361,11 @@ public class SDIManager {
public void deleteContent(GeoServerContent toDelete) throws IllegalArgumentException, MalformedURLException, RemoteServiceException {
log.debug("Deleting geoserver layer "+toDelete);
//delete layer
GeoServerRESTPublisher publisher=gis.getCurrentGeoServer().getPublisher();
GeoServerRESTPublisher publisher=currentGeoserver.getPublisher();
//delete store
publisher.removeDatastore(toDelete.getWorkspace(), toDelete.getStore(), true);
//delete WS if empty
GeoServerRESTReader reader=gis.getCurrentGeoServer().getReader();
GeoServerRESTReader reader=currentGeoserver.getReader();
if(reader.getDatastores(toDelete.getWorkspace()).isEmpty()) {
log.debug("Deleting emtpy workspace "+toDelete.getWorkspace());
publisher.removeWorkspace(toDelete.getWorkspace(), true);

8
src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java
Unescape Escape View File

@ -1,6 +1,7 @@
package org.gcube.application.geoportal.service.engine.mongo;
import java.io.IOException;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@ -49,7 +50,6 @@ public class ConcessioniMongoManager extends MongoManager{
public ConcessioniMongoManager() throws ConfigurationException {
super();
// TODO Auto-generated constructor stub
}
private static final String collectionName="legacyConcessioni";
private static final String DB_NAME="gna_dev";
@ -183,10 +183,10 @@ public class ConcessioniMongoManager extends MongoManager{
ValidationReport report= new ValidationReport("Index Report ");
PostgisIndex index;
try {
index = new PostgisIndex(record);
index.registerCentroid();
index = new PostgisIndex();
index.registerCentroid(record);
report.addMessage(ValidationStatus.PASSED, "Registered centroid");
} catch (SDIInteractionException | PublishException e) {
} catch (SDIInteractionException | PublishException | SQLException | ConfigurationException e) {
log.error("Unable to index {} ",record,e);
report.addMessage(ValidationStatus.WARNING, "Internal error while indexing.");
}

44
src/main/java/org/gcube/application/geoportal/service/engine/mongo/PostgisIndex.java
Unescape Escape View File

@ -35,29 +35,39 @@ public class PostgisIndex {
}
}
@NonNull
@Getter
private Concessione record;
private SDIManager sdiManager;
public PostgisIndex(Concessione record) throws SDIInteractionException {
public PostgisIndex() throws SDIInteractionException, SQLException, ConfigurationException {
super();
this.record = record;
this.sdiManager=new SDIManager();
init();
}
protected PostgisTable getCentroidsTable() {
return DBConstants.Concessioni.CENTROIDS;
}
public void registerCentroid() throws PublishException{
public void init() throws SQLException, ConfigurationException, SDIInteractionException {
log.debug("Contacting postgis DB .. ");
PostgisDBManagerI db=PostgisDBManager.get();
log.debug("Checking if centroids table exists..");
PostgisTable table=getCentroidsTable();
db.create(table);
db.commit();
sdiManager.configureCentroidLayer("centroids_concessioni", "gna", "gna_postgis",table,db.getConnectionDescriptor());
}
public void registerCentroid(Concessione record) throws PublishException{
try {
log.debug("Evaluating Centroid");
Map<String,String> centroidRow=evaluateCentroid();
Map<String,String> centroidRow=evaluateCentroid(record);
log.debug("Contacting postgis DB .. ");
PostgisDBManagerI db=PostgisDBManager.get();
@ -74,31 +84,21 @@ public class PostgisIndex {
ps.executeUpdate();
db.commit();
initCentroidLayer();
}catch(SQLException e) {
log.warn("Unable to publish Centroid for record "+record,e);
throw new PublishException("Unable to publish centroid.",e, null);
}catch(SDIInteractionException e) {
log.warn("Unable to publish Centroid Layer for record type "+getRecord().getRecordType(),e);
throw new PublishException("Unable to publish centroid.",e, null);
} catch (ConfigurationException e) {
log.warn("Unable to contact centroids db "+getRecord().getRecordType(),e);
log.warn("Unable to contact centroids db "+record.getRecordType(),e);
throw new PublishException("Unable to publish centroid.",e, null);
}
}
protected void initCentroidLayer() throws SDIInteractionException {
log.debug("Checking for centroid layer configuration.. ");
sdiManager.configureCentroidLayer("centroids_concessioni", "gna", "gna_postgis");
}
public void removeCentroid() {
public void removeCentroid(Concessione record) {
try {
PostgisDBManagerI db=PostgisDBManager.get();
PostgisTable centroidsTable=getCentroidsTable();
@ -110,7 +110,7 @@ public class PostgisIndex {
}
protected Map<String,String> evaluateCentroid(){
protected static Map<String,String> evaluateCentroid(Concessione record){
// CENTROID

3
src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java
Unescape Escape View File

@ -172,8 +172,9 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{
// Immagini
Concessione published=publish(target, c);
System.out.println("Published : "+published);
System.out.println("Report is : "+published.getReport());
assertNotNull(published.getReport());
assertEquals(published.getReport().getStatus(),ValidationStatus.PASSED);
assertEquals(ValidationStatus.PASSED,published.getReport().getStatus());
assertEquals(published.getImmaginiRappresentative().size(),2);
assertEquals(published.getPianteFineScavo().size(),1);

43
src/test/java/org/gcube/application/geoportal/service/SDITests.java
Unescape Escape View File

@ -0,0 +1,43 @@
package org.gcube.application.geoportal.service;
import org.gcube.application.geoportal.model.fault.ConfigurationException;
import org.gcube.application.geoportal.service.engine.SDIManager;
import org.gcube.application.geoportal.service.engine.mongo.PostgisIndex;
import org.gcube.application.geoportal.service.legacy.TokenSetter;
import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import javax.validation.constraints.AssertTrue;
import java.sql.SQLException;
import java.util.regex.Matcher;
public class SDITests {
@Before
public void init(){
TokenSetter.set("/gcube/devsec/devVRE");
}
@Test
public void registerCentroidsLayer() throws SDIInteractionException, SQLException, ConfigurationException {
PostgisIndex index=new PostgisIndex();
}
@Test
public void testRegexp(){
Matcher hostMatcher=SDIManager.HOSTNAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db");
Assert.assertTrue(hostMatcher.find());
System.out.println("HOST :\t"+hostMatcher.group());
Matcher portMatcher=SDIManager.PORT_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db");
Assert.assertTrue(portMatcher.find());
System.out.println("PORT :\t"+portMatcher.group());
Matcher dbMatcher=SDIManager.DB_NAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db");
Assert.assertTrue(dbMatcher.find());
System.out.println("DB :\t"+dbMatcher.group());
}
}