diff --git a/CHANGELOG.md b/CHANGELOG.md index c1de2f5..9f8a787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [v1.0.5-SNAPSHOT] 2021-07-23 Upgrade to gcube-smartgears-bom 2.1.0 +Fix register postgis table layer +Added PostgisIndexRecordManager ## [v1.0.4] 2020-11-11 Mongo integration with Concessione diff --git a/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java b/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java index 54e3526..978101c 100644 --- a/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java +++ b/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java @@ -1,15 +1,23 @@ package org.gcube.application.geoportal.service.engine; -import java.io.File; -import java.net.MalformedURLException; -import java.net.URL; - -import org.gcube.application.geoportal.common.model.legacy.BBOX; -import org.gcube.application.geoportal.common.model.legacy.GeoServerContent; -import org.gcube.application.geoportal.common.model.legacy.PersistedContent; -import org.gcube.application.geoportal.common.model.legacy.SDILayerDescriptor; -import org.gcube.application.geoportal.common.model.legacy.WorkspaceContent; +import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; +import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod; +import it.geosolutions.geoserver.rest.GeoServerRESTReader; +import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; +import it.geosolutions.geoserver.rest.decoder.RESTLayer; +import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder; +import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder; +import it.geosolutions.geoserver.rest.encoder.feature.FeatureTypeAttribute; +import it.geosolutions.geoserver.rest.encoder.feature.GSAttributeEncoder; +import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.utils.Files; +import org.gcube.application.geoportal.model.db.DBConstants; +import org.gcube.application.geoportal.model.db.DatabaseConnection; +import org.gcube.application.geoportal.model.db.PostgisTable; +import org.gcube.application.geoportal.service.engine.mongo.PostgisIndex; import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; import org.gcube.common.storagehub.client.dsl.FileContainer; import org.gcube.data.transfer.library.DataTransferClient; @@ -21,43 +29,52 @@ import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.spatial.data.gis.GISInterface; import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; -import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; -import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod; -import it.geosolutions.geoserver.rest.GeoServerRESTReader; -import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; -import it.geosolutions.geoserver.rest.decoder.RESTLayer; -import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder; -import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder; -import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; +import java.io.File; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @Slf4j public class SDIManager { - static private String DEFAULT_CRS="EPSG:4326"; + static private final String EPSG_4326="EPSG:4326"; + static private final String WGS84_FULL="GEOGCS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", SPHEROID[\"WGS 84\", 6378137.0, 298.257223563, AUTHORITY[\"EPSG\",\"7030\"]],"+ + "AUTHORITY[\"EPSG\",\"6326\"]], PRIMEM[\"Greenwich\", 0.0, AUTHORITY[\"EPSG\",\"8901\"]], UNIT[\"degree\", 0.017453292519943295],"+ + "AXIS[\"Geodetic longitude\", EAST], AXIS[\"Geodetic latitude\", NORTH], AUTHORITY[\"EPSG\",\"4326\"]]"; - private GISInterface gis; + public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); + public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+"); + public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); + + + private final GISInterface gis; @Getter - private DataTransferClient dtGeoServer; - private String geoserverHostName; + private final DataTransferClient dtGeoServer; + private final String geoserverHostName; + + private final AbstractGeoServerDescriptor currentGeoserver; public SDIManager() throws SDIInteractionException { try{ log.debug("Initializing GIS Interface.."); gis=GISInterface.get(); - AbstractGeoServerDescriptor geoserver=gis.getCurrentGeoServer(); - if(geoserver==null) + currentGeoserver=gis.getCurrentGeoServer(); + if(currentGeoserver==null) throw new Exception("Unable to contact data transfer for geoserver "); - log.debug("Found geoserver descriptor "+geoserver); - geoserverHostName=new URL(gis.getCurrentGeoServer().getUrl()).getHost(); + log.debug("Found geoserver descriptor "+currentGeoserver); + geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); dtGeoServer=DataTransferClient.getInstanceByEndpoint("http://"+geoserverHostName); - if(!gis.getCurrentGeoServer().getReader().existGeoserver()) + if(!currentGeoserver.getReader().existGeoserver()) throw new Exception("Geoserver not reachable"); }catch(Exception e) { throw new SDIInteractionException("Unable to initialize SDI Manager",e); @@ -71,9 +88,9 @@ public class SDIManager { public String createWorkspace(String toCreate) throws SDIInteractionException { try { - if(!gis.getCurrentGeoServer().getReader().getWorkspaceNames().contains(toCreate)) { + if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { log.debug("Creating workspace : "+toCreate); - if(!gis.getCurrentGeoServer().getPublisher().createWorkspace(toCreate)) + if(!currentGeoserver.getPublisher().createWorkspace(toCreate)) throw new SDIInteractionException("Unable to create workspace "+toCreate); }else log.debug("Workspace "+toCreate+" exists."); return toCreate; @@ -149,18 +166,19 @@ public class SDIManager { String toSetLayerName=filename; int count=0; - while(gis.getCurrentGeoServer().getReader().getLayer(workspace,toSetLayerName)!=null){ + GeoServerRESTReader gsReader=currentGeoserver.getReader(); + while(gsReader.getLayer(workspace,toSetLayerName)!=null){ count++; toSetLayerName=filename+"_"+count; log.debug("layer for "+filename+" already existing, trying "+toSetLayerName); - }; - - + } + + String storeName=toSetLayerName+"_store"; content.setStore(storeName); content.setFeatureType(toSetLayerName); - GeoServerRESTPublisher publisher=gis.getCurrentGeoServer().getPublisher(); + GeoServerRESTPublisher publisher=currentGeoserver.getPublisher(); log.debug("Trying to create remote workspace : "+workspace); createWorkspace(workspace); @@ -179,7 +197,7 @@ public class SDIManager { // UploadMethod.FILE, // neeeds zip UploadMethod.EXTERNAL, // needs shp directoryPath.toURI(), - DEFAULT_CRS, //SRS + EPSG_4326, //SRS ""); // default style if(!published) { @@ -187,9 +205,9 @@ public class SDIManager { } currentElement.setLayerName(toSetLayerName); - GeoServerRESTReader reader=gis.getCurrentGeoServer().getReader(); - RESTLayer l=reader.getLayer(workspace, toSetLayerName); - RESTFeatureType f= reader.getFeatureType(l); + + RESTLayer l=gsReader.getLayer(workspace, toSetLayerName); + RESTFeatureType f= gsReader.getFeatureType(l); /*http://geoserver1.dev.d4science.org/geoserver/gna_conc_18/wms? service=WMS&version=1.1.0&request=GetMap&layers=gna_conc_18:pos& styles=&bbox=8.62091913167495,40.62975046683799,8.621178639172953,40.630257904721645& @@ -207,7 +225,7 @@ public class SDIManager { f.getMinY(), f.getMaxX(), f.getMaxY(), - DEFAULT_CRS, + EPSG_4326, 400, 400)); @@ -227,7 +245,51 @@ public class SDIManager { } } - private String createStoreFromPostgisDB(String workspace,String storeName) throws SDIInteractionException { + private String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException { + String storeName=encoder.getName(); + try { + log.debug("Looking for datastore "+storeName+" under "+workspace); + + if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null) + + if(!currentGeoserver.getDataStoreManager().create(workspace, encoder)) + throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace); + log.debug("Store "+storeName+" exists under "+workspace); + return storeName; + } catch (IllegalArgumentException | MalformedURLException e) { + throw new SDIInteractionException("Unable to create store "+storeName,e); + } + } + + private String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{ + String connectionUrl=connection.getUrl(); + + Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl); + if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection); + + Matcher port = PORT_PATTERN.matcher(connectionUrl); + if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection); + + Matcher db = DB_NAME_PATTERN.matcher(connectionUrl); + if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection); + + + GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); + encoder.setHost(hostname.group()); + encoder.setPort(Integer.parseInt(port.group())); + encoder.setDatabase(db.group()); + encoder.setSchema("public"); + encoder.setUser(connection.getUser()); + encoder.setPassword(connection.getPwd()); + encoder.setLooseBBox(true); + encoder.setDatabaseType("postgis"); + encoder.setEnabled(true); + encoder.setFetchSize(1000); + encoder.setValidateConnections(true); + + return createStore(encoder,workspace); + } + private String createStoreFromJNDIDB(String workspace,String storeName) throws SDIInteractionException { //SET BY PROVISIONING GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); encoder.setJndiReferenceName("java:comp/env/jdbc/postgres"); @@ -236,26 +298,14 @@ public class SDIManager { encoder.setEnabled(true); encoder.setFetchSize(1000); encoder.setValidateConnections(true); - try { - log.debug("Looking for datastore "+storeName+" under "+workspace); - - if(gis.getCurrentGeoServer().getReader().getDatastore(workspace,storeName)==null) - - if(!gis.getCurrentGeoServer().getDataStoreManager().create(workspace, encoder)) - throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace); - log.debug("Store "+storeName+" exists under "+workspace); - return storeName; - } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create store "+storeName,e); - } - + return createStore(encoder,workspace); } private String publishStyle(File sldFile,String name) throws SDIInteractionException { try { - if(!gis.getCurrentGeoServer().getReader().existsStyle(name)) { + if(!currentGeoserver.getReader().existsStyle(name)) { log.debug("Registering style "+name); - if(!gis.getCurrentGeoServer().getPublisher().publishStyle(sldFile, name)) + if(!currentGeoserver.getPublisher().publishStyle(sldFile, name)) throw new SDIInteractionException("Unable to register style "+name); }else log.debug("Style "+name+" already existing"); return name; @@ -265,15 +315,21 @@ public class SDIManager { } - public String configureCentroidLayer(String name,String workspace,String storeName) throws SDIInteractionException { + public String configureCentroidLayer(String name,String workspace,String storeName,PostgisTable table, DatabaseConnection connection) throws SDIInteractionException { GSFeatureTypeEncoder fte=new GSFeatureTypeEncoder(); fte.setAbstract("Centroid layer for "+name); fte.setEnabled(true); - fte.setNativeCRS(DEFAULT_CRS); + fte.setNativeCRS(WGS84_FULL); fte.setTitle(name); fte.setName(name); - + + + // GeoServer loads all fields +// fte.setAttribute(attrs); + + + fte.setLatLonBoundingBox(-180.0, -90.0, 180.0, 90.0, WGS84_FULL); String style="clustered_centroids"; @@ -285,11 +341,12 @@ public class SDIManager { //Checking workspace createWorkspace(workspace); //Checking store - createStoreFromPostgisDB(workspace, storeName); - //Checkig layer + createStoreFromPostgisDB(workspace, storeName,connection); + //Checking layer publishStyle(Files.getFileFromResources("styles/clustered_points.sld"),style); - if(gis.getCurrentGeoServer().getReader().getLayer(workspace, name)==null) - if(!gis.getCurrentGeoServer().getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder)) + + if(currentGeoserver.getReader().getLayer(workspace, name)==null) + if(!currentGeoserver.getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder)) throw new SDIInteractionException("Unable to create layer "+name); log.debug("layer "+name+" already exists"); return name; @@ -304,11 +361,11 @@ public class SDIManager { public void deleteContent(GeoServerContent toDelete) throws IllegalArgumentException, MalformedURLException, RemoteServiceException { log.debug("Deleting geoserver layer "+toDelete); //delete layer - GeoServerRESTPublisher publisher=gis.getCurrentGeoServer().getPublisher(); + GeoServerRESTPublisher publisher=currentGeoserver.getPublisher(); //delete store publisher.removeDatastore(toDelete.getWorkspace(), toDelete.getStore(), true); //delete WS if empty - GeoServerRESTReader reader=gis.getCurrentGeoServer().getReader(); + GeoServerRESTReader reader=currentGeoserver.getReader(); if(reader.getDatastores(toDelete.getWorkspace()).isEmpty()) { log.debug("Deleting emtpy workspace "+toDelete.getWorkspace()); publisher.removeWorkspace(toDelete.getWorkspace(), true); diff --git a/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java b/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java index 7f0eeb0..c061c8b 100644 --- a/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java +++ b/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java @@ -1,6 +1,7 @@ package org.gcube.application.geoportal.service.engine.mongo; import java.io.IOException; +import java.sql.SQLException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -49,7 +50,6 @@ public class ConcessioniMongoManager extends MongoManager{ public ConcessioniMongoManager() throws ConfigurationException { super(); - // TODO Auto-generated constructor stub } private static final String collectionName="legacyConcessioni"; private static final String DB_NAME="gna_dev"; @@ -183,10 +183,10 @@ public class ConcessioniMongoManager extends MongoManager{ ValidationReport report= new ValidationReport("Index Report "); PostgisIndex index; try { - index = new PostgisIndex(record); - index.registerCentroid(); + index = new PostgisIndex(); + index.registerCentroid(record); report.addMessage(ValidationStatus.PASSED, "Registered centroid"); - } catch (SDIInteractionException | PublishException e) { + } catch (SDIInteractionException | PublishException | SQLException | ConfigurationException e) { log.error("Unable to index {} ",record,e); report.addMessage(ValidationStatus.WARNING, "Internal error while indexing."); } diff --git a/src/main/java/org/gcube/application/geoportal/service/engine/mongo/PostgisIndex.java b/src/main/java/org/gcube/application/geoportal/service/engine/mongo/PostgisIndex.java index 0c6d2af..8aed125 100644 --- a/src/main/java/org/gcube/application/geoportal/service/engine/mongo/PostgisIndex.java +++ b/src/main/java/org/gcube/application/geoportal/service/engine/mongo/PostgisIndex.java @@ -35,29 +35,39 @@ public class PostgisIndex { } } - @NonNull - @Getter - private Concessione record; + private SDIManager sdiManager; - public PostgisIndex(Concessione record) throws SDIInteractionException { + public PostgisIndex() throws SDIInteractionException, SQLException, ConfigurationException { super(); - this.record = record; this.sdiManager=new SDIManager(); + init(); } protected PostgisTable getCentroidsTable() { return DBConstants.Concessioni.CENTROIDS; } - - - public void registerCentroid() throws PublishException{ + + public void init() throws SQLException, ConfigurationException, SDIInteractionException { + log.debug("Contacting postgis DB .. "); + PostgisDBManagerI db=PostgisDBManager.get(); + log.debug("Checking if centroids table exists.."); + PostgisTable table=getCentroidsTable(); + db.create(table); + db.commit(); + sdiManager.configureCentroidLayer("centroids_concessioni", "gna", "gna_postgis",table,db.getConnectionDescriptor()); + + } + + + + public void registerCentroid(Concessione record) throws PublishException{ try { log.debug("Evaluating Centroid"); - Map centroidRow=evaluateCentroid(); + Map centroidRow=evaluateCentroid(record); log.debug("Contacting postgis DB .. "); PostgisDBManagerI db=PostgisDBManager.get(); @@ -74,31 +84,21 @@ public class PostgisIndex { ps.executeUpdate(); db.commit(); - initCentroidLayer(); - }catch(SQLException e) { log.warn("Unable to publish Centroid for record "+record,e); throw new PublishException("Unable to publish centroid.",e, null); - }catch(SDIInteractionException e) { - log.warn("Unable to publish Centroid Layer for record type "+getRecord().getRecordType(),e); - throw new PublishException("Unable to publish centroid.",e, null); } catch (ConfigurationException e) { - log.warn("Unable to contact centroids db "+getRecord().getRecordType(),e); + log.warn("Unable to contact centroids db "+record.getRecordType(),e); throw new PublishException("Unable to publish centroid.",e, null); } } - protected void initCentroidLayer() throws SDIInteractionException { - log.debug("Checking for centroid layer configuration.. "); - - sdiManager.configureCentroidLayer("centroids_concessioni", "gna", "gna_postgis"); - } - public void removeCentroid() { + public void removeCentroid(Concessione record) { try { PostgisDBManagerI db=PostgisDBManager.get(); PostgisTable centroidsTable=getCentroidsTable(); @@ -110,7 +110,7 @@ public class PostgisIndex { } - protected Map evaluateCentroid(){ + protected static Map evaluateCentroid(Concessione record){ // CENTROID diff --git a/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java b/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java index 4b64d40..663a332 100644 --- a/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java +++ b/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java @@ -172,8 +172,9 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ // Immagini Concessione published=publish(target, c); System.out.println("Published : "+published); + System.out.println("Report is : "+published.getReport()); assertNotNull(published.getReport()); - assertEquals(published.getReport().getStatus(),ValidationStatus.PASSED); + assertEquals(ValidationStatus.PASSED,published.getReport().getStatus()); assertEquals(published.getImmaginiRappresentative().size(),2); assertEquals(published.getPianteFineScavo().size(),1); diff --git a/src/test/java/org/gcube/application/geoportal/service/SDITests.java b/src/test/java/org/gcube/application/geoportal/service/SDITests.java new file mode 100644 index 0000000..37b3444 --- /dev/null +++ b/src/test/java/org/gcube/application/geoportal/service/SDITests.java @@ -0,0 +1,43 @@ +package org.gcube.application.geoportal.service; + +import org.gcube.application.geoportal.model.fault.ConfigurationException; +import org.gcube.application.geoportal.service.engine.SDIManager; +import org.gcube.application.geoportal.service.engine.mongo.PostgisIndex; +import org.gcube.application.geoportal.service.legacy.TokenSetter; +import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.validation.constraints.AssertTrue; +import java.sql.SQLException; +import java.util.regex.Matcher; + +public class SDITests { + + + @Before + public void init(){ + TokenSetter.set("/gcube/devsec/devVRE"); + } + + @Test + public void registerCentroidsLayer() throws SDIInteractionException, SQLException, ConfigurationException { + PostgisIndex index=new PostgisIndex(); + } + + @Test + public void testRegexp(){ + Matcher hostMatcher=SDIManager.HOSTNAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); + Assert.assertTrue(hostMatcher.find()); + System.out.println("HOST :\t"+hostMatcher.group()); + + Matcher portMatcher=SDIManager.PORT_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); + Assert.assertTrue(portMatcher.find()); + System.out.println("PORT :\t"+portMatcher.group()); + + Matcher dbMatcher=SDIManager.DB_NAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); + Assert.assertTrue(dbMatcher.find()); + System.out.println("DB :\t"+dbMatcher.group()); + } +}