From 2a413527484d558a4dabca71e14550ee291f101c Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Fri, 18 Feb 2022 18:11:12 +0100 Subject: [PATCH] SDI Indexing --- .../plugins/ConcessioniLifeCycleManager.java | 56 +++- .../cms/concessioni/sdi/SDIManager.java | 282 ---------------- .../cms/sdi/engine/DBConstants.java | 139 ++++++++ .../cms/sdi/engine/PostgisDBManager.java | 148 ++++++++ .../cms/sdi/engine/PostgisDBManagerI.java | 38 +++ .../cms/sdi/engine/PostgisIndexer.java | 105 ++++++ .../cms/sdi/engine/PostgisTable.java | 316 ++++++++++++++++++ .../cms/sdi/engine/SDIManager.java | 180 ++++++++++ .../cms/sdi/engine/SDIManagerWrapper.java | 257 ++++++++++++++ .../cms/sdi/faults/DataParsingException.java | 26 ++ .../sdi/faults/SDIInteractionException.java | 2 +- .../sdi/model/SDIConfiguration.java | 2 +- .../cms/sdi/plugins/SDIAbstractPlugin.java | 82 +++++ .../cms/sdi/plugins/SDIIndexerPlugin.java | 214 ++++++++++++ .../plugins/SDIMaterializerPlugin.java | 29 +- 15 files changed, 1571 insertions(+), 305 deletions(-) delete mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/SDIManager.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/DBConstants.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManager.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManagerI.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/faults/DataParsingException.java rename concessioni-lifecycle/src/main/java/org/gcube/application/cms/{concessioni => }/sdi/faults/SDIInteractionException.java (90%) rename concessioni-lifecycle/src/main/java/org/gcube/application/cms/{concessioni => }/sdi/model/SDIConfiguration.java (84%) create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java create mode 100644 concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java rename concessioni-lifecycle/src/main/java/org/gcube/application/cms/{concessioni => sdi}/plugins/SDIMaterializerPlugin.java (88%) diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java index 373c2df..2e9c422 100644 --- a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/ConcessioniLifeCycleManager.java @@ -6,14 +6,13 @@ import org.bson.BsonDocument; import org.bson.BsonString; import org.bson.BsonValue; import org.bson.Document; -import org.gcube.application.cms.concessioni.sdi.SDIManager; -import org.gcube.application.cms.concessioni.sdi.faults.SDIInteractionException; -import org.gcube.application.cms.plugins.InitializablePlugin; +import org.gcube.application.cms.plugins.IndexerPluginInterface; import org.gcube.application.cms.plugins.MaterializationPlugin; import org.gcube.application.cms.plugins.PluginManagerInterface; import org.gcube.application.cms.plugins.faults.EventException; import org.gcube.application.cms.plugins.reports.*; import org.gcube.application.cms.plugins.requests.EventExecutionRequest; +import org.gcube.application.cms.plugins.requests.IndexDocumentRequest; import org.gcube.application.cms.plugins.requests.MaterializationRequest; import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione; @@ -38,6 +37,11 @@ import org.gcube.application.geoportal.common.utils.Files; @Slf4j public class ConcessioniLifeCycleManager implements LifecycleManager { + private static class Phases { + public static final String PENDING_APPROVAL="Pending Approval"; + } + + private static final PluginDescriptor DESCRIPTOR=new PluginDescriptor("GNA-CONCESSIONI-LC", PluginDescriptor.BaseTypes.LIFECYCLE_MANAGER); static { DESCRIPTOR.setDescription("GNA Concessioni. This plugin supports custom lifecycle management for the GNA Concessioni UseCase."); @@ -50,6 +54,8 @@ public class ConcessioniLifeCycleManager implements LifecycleManager { // TODO DISCOVER MaterializationPlugin plugin; + IndexerPluginInterface indexerPlugin; + @Override public InitializationReport initInContext() throws InitializationException { InitializationReport report = new InitializationReport(); @@ -69,7 +75,7 @@ public class ConcessioniLifeCycleManager implements LifecycleManager { try{ report.setStatus(Report.Status.OK); plugin= (MaterializationPlugin) pluginManager.getById("SDI-Default-Materializer"); - if(plugin == null ) throw new InitializationException("SDI-Default-Materializer not found"); + indexerPlugin = (IndexerPluginInterface) pluginManager.getById("SDI-Indexer-Plugin"); } catch (Throwable e) { log.error("Unable to initialize plugins ",e); report.setStatus(Report.Status.WARNING); @@ -114,11 +120,14 @@ public class ConcessioniLifeCycleManager implements LifecycleManager { MaterializationReport matRep = plugin.materialize(matReq); + report.setResultingDocument(matRep.getResultingDocument()); switch(matRep.getStatus()){ case OK : { - info.setPhase("Pending Approval"); + info.setPhase(Phases.PENDING_APPROVAL); //TODO Index-confidential + + //TODO Notifications break; @@ -145,6 +154,38 @@ public class ConcessioniLifeCycleManager implements LifecycleManager { case "APPROVE DRAFT":{ // Index-published + + if(!request.getDocument().getLifecycleInformation().getPhase() + .equals(Phases.PENDING_APPROVAL)) + throw new StepException("Document is not in "+Phases.PENDING_APPROVAL+" phase"); + + IndexDocumentRequest indexRequest = new IndexDocumentRequest(); + indexRequest.setDocument(request.getDocument()); + indexRequest.setProfile(request.getProfile()); + Document callParameters = new Document(); + callParameters.put("workspace","gna_concessioni_"+request.getContext()); + callParameters.put("indexName","gna_concessioni_centroids_"+request.getContext()); + indexRequest.setCallParameters(callParameters); + + IndexDocumentReport indexReport = indexerPlugin.index(indexRequest); + + switch(indexReport.getStatus()){ + case OK : { + info.setPhase("PUBLISHED"); + + break; + } + case ERROR : { + info.setLastOperationStatus(LifecycleInformation.Status.ERROR); + indexReport.getMessages().forEach(s -> info.addErrorMessage(s)); + break; + } + case WARNING : { + info.setLastOperationStatus(LifecycleInformation.Status.WARNING); + indexReport.getMessages().forEach(s -> info.addWarningMessage(s)); + break; + } + } break; } @@ -170,6 +211,11 @@ public class ConcessioniLifeCycleManager implements LifecycleManager { log.info("Executing Event {}",request); EventExecutionReport report=new EventExecutionReport(); report.setTheRequest(request); + if(report.getToSetLifecycleInformation()==null){ + LifecycleInformation lc=new LifecycleInformation(); + report.setStatus(Report.Status.OK); + report.setToSetLifecycleInformation(lc); + } TriggeredEvents info=new TriggeredEvents(); diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/SDIManager.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/SDIManager.java deleted file mode 100644 index 375578b..0000000 --- a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/SDIManager.java +++ /dev/null @@ -1,282 +0,0 @@ -package org.gcube.application.cms.concessioni.sdi; - -import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; -import it.geosolutions.geoserver.rest.GeoServerRESTReader; -import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; -import it.geosolutions.geoserver.rest.decoder.RESTLayer; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.bson.Document; -import org.gcube.application.cms.concessioni.sdi.faults.SDIInteractionException; -import org.gcube.application.cms.plugins.requests.BaseExecutionRequest; -import org.gcube.application.cms.serialization.Serialization; -import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer; - -import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile; -import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; - -import org.gcube.application.geoportal.common.utils.Files; - - - -import org.gcube.data.transfer.library.DataTransferClient; -import org.gcube.data.transfer.library.TransferResult; -import org.gcube.data.transfer.model.Destination; -import org.gcube.data.transfer.model.DestinationClashPolicy; -import org.gcube.spatial.data.gis.GISInterface; -import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -@Slf4j -public class SDIManager { - - - static private final String EPSG_4326="EPSG:4326"; - static private final String WGS84_FULL="GEOGCS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", SPHEROID[\"WGS 84\", 6378137.0, 298.257223563, AUTHORITY[\"EPSG\",\"7030\"]],"+ - "AUTHORITY[\"EPSG\",\"6326\"]], PRIMEM[\"Greenwich\", 0.0, AUTHORITY[\"EPSG\",\"8901\"]], UNIT[\"degree\", 0.017453292519943295],"+ - "AXIS[\"Geodetic longitude\", EAST], AXIS[\"Geodetic latitude\", NORTH], AUTHORITY[\"EPSG\",\"4326\"]]"; - - - public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); - public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+"); - public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); - - - private final GISInterface gis; - @Getter - private final DataTransferClient dtGeoServer; - @Getter - private final String geoserverHostName; - @Getter - private final AbstractGeoServerDescriptor currentGeoserver; - - - public SDIManager() throws SDIInteractionException { - try{ - log.debug("Initializing GIS Interface.."); - gis=GISInterface.get(); - currentGeoserver=gis.getCurrentGeoServer(); - if(currentGeoserver==null) - throw new Exception("Unable to contact data transfer for geoserver "); - - log.debug("Found geoserver descriptor "+currentGeoserver); - geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); - - log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); - dtGeoServer=DataTransferClient.getInstanceByEndpoint("https://"+geoserverHostName); - if(!currentGeoserver.getReader().existGeoserver()) - throw new Exception("Geoserver not reachable"); - }catch(Exception e) { - throw new SDIInteractionException("Unable to initialize SDI Manager",e); - } - } - - /** - * Expected paramters : - * - "workspace" - * - "layerTitle" - * - "documentID" - * - "basePersistencePath" (profile specific, e.g. "GNA") - * - * @param fileSet - * @param params - * @return - * @throws SDIInteractionException - */ - public RegisteredFileSet materializeLayer(RegisteredFileSet fileSet, Document params) throws SDIInteractionException{ - try { - log.debug("Materializing FS {} on {} ", fileSet, geoserverHostName); - - // validate parameters - String workspace = BaseExecutionRequest.getMandatory("workspace", params); - String documentID = BaseExecutionRequest.getMandatory("documentID", params); - String basePersistencePAth = BaseExecutionRequest.getMandatory("basePersistencePath", params); - - - // check if empty - if (fileSet.getPayloads().isEmpty()) throw new SDIInteractionException("No payload to materialize"); - - - Document geoserverInfo = new Document(); - geoserverInfo.put("_type", "Geoserver"); - geoserverInfo.put("workspace", workspace); - - - // Evaluate Layer Name - String baseName = getToUseBaseLayerName(fileSet); - log.debug("Base layer name is {}, checking conflicts.. "); - String toSetLayerName = baseName; - //Check if layer already exists - int count = 0; - GeoServerRESTReader gsReader = currentGeoserver.getReader(); - while (gsReader.getLayer(workspace, toSetLayerName) != null) { - count++; - toSetLayerName = baseName + "_" + count; - log.debug("layer for " + baseName + " already existing, trying " + toSetLayerName); - } - geoserverInfo.put("layerName", toSetLayerName); - log.debug("Layer name will be {} ", toSetLayerName); - - - String folderRelativePath = basePersistencePAth + "/" + documentID + "/" + fileSet.getUUID() + "/" + toSetLayerName; - log.debug("GS Relative destination path is {} ", folderRelativePath); - geoserverInfo.put("persistencePath", folderRelativePath); - - List filenames = new ArrayList<>(); - - String absolutePath = null; - - for (Object o : fileSet.getPayloads()) { - RegisteredFile file = Serialization.convert(o, RegisteredFile.class); - log.info("Sending {} to GS {} at {} ", file, geoserverHostName, folderRelativePath); - String completeFilename = Files.fixFilename(file.getName()); - completeFilename = completeFilename.replaceAll(baseName, toSetLayerName); - - - Destination destination = new Destination(completeFilename); - destination.setCreateSubfolders(true); - destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); - destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); - - destination.setPersistenceId("geoserver"); - destination.setSubFolder(folderRelativePath); - - log.debug("Sending {} to {}", file, destination); - TransferResult result = getDtGeoServer().httpSource(new URL(file.getLink()), destination); - log.debug("Transferred " + result); - - - filenames.add(completeFilename); - // NB Clash con subfolder is APPEND, thus FOLDER is expected to be the one specified by caller - //geoserverInfo.put(""result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/"))); - absolutePath = result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/")); - } - geoserverInfo.put("files", filenames); - - // Publishing layer in GS - String storeName = toSetLayerName + "_store"; - geoserverInfo.put("storeName", storeName); - - GeoServerRESTPublisher publisher = currentGeoserver.getPublisher(); - log.debug("Trying to create remote workspace : " + workspace); - createWorkspace(workspace); - - - log.debug("Publishing remote folder " + absolutePath); - - URL directoryPath = new URL("file:" + absolutePath + "/" + toSetLayerName + ".shp"); - - //TODO Evaluate SRS - - boolean published = publisher.publishShp( - workspace, - storeName, - null, - toSetLayerName, - // UploadMethod.FILE, // neeeds zip - GeoServerRESTPublisher.UploadMethod.EXTERNAL, // needs shp - directoryPath.toURI(), - EPSG_4326, //SRS - ""); // default style - - if (!published) { - throw new SDIInteractionException("Unable to publish layer " + toSetLayerName + " under " + workspace + ". Unknown Geoserver fault."); - } - - RESTLayer l = gsReader.getLayer(workspace, toSetLayerName); - RESTFeatureType f = gsReader.getFeatureType(l); - /*https://geoserver1.dev.d4science.org/geoserver/gna_conc_18/wms? - service=WMS&version=1.1.0&request=GetMap&layers=gna_conc_18:pos& - styles=&bbox=8.62091913167495,40.62975046683799,8.621178639172953,40.630257904721645& - width=392&height=768&srs=EPSG:4326&format=application/openlayers */ - - - List ogcLinks = new ArrayList<>(); - - Document wmsLink = new Document(); - wmsLink.put("wms", String.format("https://%1$s/geoserver/%2$s/wms?" - + "service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&" - + "styles=&bbox=%4$f,%5$f,%6$f,%7$f&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d", - geoserverHostName, - workspace, - toSetLayerName, - f.getMinX(), - f.getMinY(), - f.getMaxX(), - f.getMaxY(), - EPSG_4326, - 400, - 400)); - ogcLinks.add(wmsLink); - - List platformInfo = new ArrayList<>(); - platformInfo.add(geoserverInfo); - // TODO Metadata - - - // Materialization object - GCubeSDILayer materialization = new GCubeSDILayer(); - materialization.put(GCubeSDILayer.OGC_LINKS, ogcLinks); - materialization.put(GCubeSDILayer.B_BOX, new GCubeSDILayer.BBOX(f.getMaxX(), f.getMaxY(), f.getMinX(), f.getMinY())); - materialization.put(GCubeSDILayer.PLATFORM_INFO, platformInfo); - - log.info("Generated Materialization {}", materialization); - - //Add Materialization to registered file set - List materializations = fileSet.getMaterializations(); - if (materializations == null) materializations = new ArrayList(); - materializations.add(materialization); - fileSet.put(RegisteredFileSet.MATERIALIZATIONS, materializations); - - return fileSet; - }catch(SDIInteractionException e){ - throw e; - }catch (Throwable t){ - throw new SDIInteractionException("Unexpected exception while trying to materialize File Set "+t.getMessage(),t); - } - } - - - public String createWorkspace(String toCreate) throws SDIInteractionException { - try { - if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { - log.debug("Creating workspace : "+toCreate); - if(!currentGeoserver.getPublisher().createWorkspace(toCreate)) - throw new SDIInteractionException("Unable to create workspace "+toCreate); - }else log.debug("Workspace "+toCreate+" exists."); - return toCreate; - } catch (IllegalArgumentException | MalformedURLException e) { - throw new SDIInteractionException("Unable to create workspace "+toCreate,e); - } - } - - - private static final String getToUseBaseLayerName(RegisteredFileSet fileset){ - // ******** IDENTIFY LAYER NAME correct layer name - // Must be unique under same WS - // equal to shp base name - - String baseName= ""; - - // Chose layer name first identifying filename - for(Object p:fileset.getPayloads()){ - RegisteredFile f = Serialization.convert(p,RegisteredFile.class); - String name=f.getName(); - if(name.endsWith(".shp")) { - log.debug("SHP is {}",name); - baseName= Files.fixFilename(name.substring(0,name.lastIndexOf('.'))); - break; - } - } - return baseName; - } - - - - -} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/DBConstants.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/DBConstants.java new file mode 100644 index 0000000..ca73ce1 --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/DBConstants.java @@ -0,0 +1,139 @@ +package org.gcube.application.cms.sdi.engine; + +import static org.gcube.application.cms.sdi.engine.PostgisTable.*; + +import java.util.ArrayList; + +public class DBConstants { + + public static enum TYPE{ + + Concessioni,Mosi,Mopr + } + + public static class Defaults{ + public static final String DEFAULT_GEOMETRY_COLUMN_NAME="geom"; + public static final String INTERNAL_ID="internal_id"; + + public static final String XCOORD_FIELD="xcoord"; + public static final String YCOORD_FIELD="ycoord"; + } + + + + /** + * nome,anno,regione,xcentroid,ycentroid,csv,shp, + * geopackage,nome_csv,nome_shp,nome_geo, + * poligono,punti,linee, + * nome_progetto, descrizione_progetto,descrizione_contenuto,autore,contributore, + * titolare_dati,responsabile,editore, + * finanziamento,soggetto, + * risorse_correlate, + * date_scavo,data_archiviazione, + * versione,licenza,titolare_licenza_copyright,accesso_dati,parole_chiave + * + * @author FabioISTI + * + */ + public static class Concessioni{ + + public static final String PRODUCT_ID="product_id"; + public static final String NOME="nome"; + public static final String ANNO="anno"; + public static final String REGIONE="regione"; + public static final String GEOMETRY= Defaults.DEFAULT_GEOMETRY_COLUMN_NAME; + + //Extension + public static final String DESCRIZIONE="descrizione"; + public static final String CONTENUTO="contenuto"; + public static final String AUTORE="autore"; + public static final String CONTRIBUTORE="contributore"; + public static final String TITOLARE="titolare"; + public static final String RESPONSABILE="responsabile"; + public static final String EDITORE="editore"; + public static final String FINANZIAMENTO="finanziamento"; + public static final String SOGGETTO="soggetto"; + public static final String RISORSE="risorse"; + public static final String DATE_SCAVO="date_scavo"; + public static final String DATA_ARCHIVIAZIONE="data_archiviazione"; + public static final String VERSIONE="versione"; + public static final String LICENZA="licenza"; + public static final String TITOLARE_LICENZA="titolare_licenza"; + public static final String ACCESSO="accesso"; + public static final String PAROLE_CHIAVE="parole_chiave"; + + + // + + + + public static final ArrayList COLUMNS=new ArrayList(); + public static final PostgisTable CENTROIDS=new PostgisTable("centroids_concessioni", + COLUMNS, GeometryType.POINT); + + static { + CENTROIDS.getFields().add(new Field(Defaults.INTERNAL_ID,FieldType.AUTOINCREMENT)); + CENTROIDS.getFields().add(new Field(PRODUCT_ID,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(NOME,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(ANNO,FieldType.INT)); + CENTROIDS.getFields().add(new Field(REGIONE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(GEOMETRY,FieldType.GEOMETRY)); + + //EXtenions + CENTROIDS.getFields().add(new Field(DESCRIZIONE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(CONTENUTO,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(AUTORE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(CONTRIBUTORE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(TITOLARE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(RESPONSABILE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(EDITORE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(FINANZIAMENTO,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(SOGGETTO,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(RISORSE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(DATE_SCAVO,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(DATA_ARCHIVIAZIONE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(VERSIONE,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(LICENZA,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(TITOLARE_LICENZA,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(ACCESSO,FieldType.TEXT)); + CENTROIDS.getFields().add(new Field(PAROLE_CHIAVE,FieldType.TEXT)); + + } + } + + public static class MOSI{ + + public static final String CODE="code"; + public static final String GEOMETRY= Defaults.DEFAULT_GEOMETRY_COLUMN_NAME; + + + public static final ArrayList COLUMNS=new ArrayList(); + public static final PostgisTable CENTROID_MOSI=new PostgisTable("centroids_mosi", + new ArrayList(), GeometryType.POINT); + + static { + CENTROID_MOSI.getFields().add(new Field(Defaults.INTERNAL_ID,FieldType.AUTOINCREMENT)); + CENTROID_MOSI.getFields().add(new Field(GEOMETRY,FieldType.GEOMETRY)); + CENTROID_MOSI.getFields().add(new Field(CODE,FieldType.TEXT)); + } + } + + public static class MOPR{ + public static final PostgisTable CENTROID_MOPR=new PostgisTable("centroids_mopr", + new ArrayList(), GeometryType.POINT); + } + + + + + public static class INTERNAL{ + public static final String DB_NAME="gna_internal_db"; + + public static final String RECORD="RECORD"; +// public static final String CONCESSIONE="CONCESSIONE"; +// public static final String + } + + + +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManager.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManager.java new file mode 100644 index 0000000..bb1becc --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManager.java @@ -0,0 +1,148 @@ +package org.gcube.application.cms.sdi.engine; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.sdi.faults.DataParsingException; +import org.gcube.application.geoportal.common.model.legacy.BBOX; +import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; +import java.sql.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +public class PostgisDBManager implements PostgisDBManagerI { + + private Connection conn=null; + + + public PostgisDBManager(Connection conn) throws SQLException { + this.conn=conn; + } + + @Override + public void create(PostgisTable toCreate) throws SQLException { + String createStmt=toCreate.getCreateStatement(); + log.debug("Executing create : "+createStmt); + conn.createStatement().executeUpdate(createStmt); + } + + + /* (non-Javadoc) + * @see org.gcube.application.geoportal.PostgisDBManagerI#commit() + */ + @Override + public void commit() throws SQLException { + conn.commit(); + } + +// /* (non-Javadoc) +// * @see org.gcube.application.geoportal.PostgisDBManagerI#evaluateBoundingBox(org.gcube.application.geoportal.model.PostgisTable) +// */ +// @Override +// public BBOX evaluateBoundingBox(PostgisTable table) throws SQLException, DataParsingException { +// ResultSet rs=conn.createStatement().executeQuery("Select ST_Extent("+table.getGeometryColumn()+") as extent from "+table.getTablename()); +// if(rs.next()) +// return DBUtils.parseST_Extent(rs.getString("extent")); +// else throw new SQLException("No extent returned"); +// } + + /* (non-Javadoc) + * @see org.gcube.application.geoportal.PostgisDBManagerI#evaluateBoundingBox(org.gcube.application.geoportal.model.PostgisTable) + */ +// @Override +// public PostgisTable.POINT evaluateCentroid(PostgisTable table) throws SQLException, DataParsingException { +// ResultSet rs=conn.createStatement().executeQuery("Select ST_AsText(ST_Centroid(ST_Collect("+table.getGeometryColumn()+"))) as centroid from "+table.getTablename()); +// if(rs.next()) +// return PostgisTable.POINT.parsePOINT(rs.getString("centroid")); +// else throw new SQLException("No extent returned"); +// } + + /* (non-Javadoc) + * @see org.gcube.application.geoportal.PostgisDBManagerI#prepareInsertStatement(org.gcube.application.geoportal.model.PostgisTable, boolean, boolean) + */ + @Override + public PreparedStatement prepareInsertStatement(PostgisTable target, boolean createTable, boolean geometryAsText) throws SQLException { + if(createTable) { + create(target); + } + String insertStmt=target.getInsertionStatement(geometryAsText); + log.debug("Preparing insert statement : "+insertStmt); + return conn.prepareStatement(insertStmt); + } + + + @Override + public int deleteByFieldValue(PostgisTable target, PostgisTable.Field field, Object value) throws SQLException { + String query=target.getDeleteByFieldStatement(field); + log.debug("Preparing DELETE SQL {} with field {} = {} ",query,field,value); + + PreparedStatement stmt = conn.prepareStatement(query); + target.setObjectInPreparedStatement(field, value, stmt, 1); + int result=stmt.executeUpdate(); + log.debug("DELETED {} rows ",result); + return result; + } + + + /* (non-Javadoc) + * @see org.gcube.application.geoportal.PostgisDBManagerI#deleteTable(java.lang.String) + */ + @Override + public void deleteTable(String tableName) throws SQLException { + conn.createStatement().executeUpdate("DROP TABLE "+tableName); + } + + /* (non-Javadoc) + * @see org.gcube.application.geoportal.PostgisDBManagerI#truncate(java.lang.String) + */ + @Override + public void truncate(String tableName) throws SQLException{ + conn.createStatement().executeUpdate("TRUNCATE Table "+tableName); + } + +// @Override +// public ResultSet queryAll(PostgisTable table) throws SQLException { +// // TODO Check schema +// return conn.createStatement().executeQuery("Select * from "+table.getTablename()); +// } + + + // *********************** INNER UTILS CLASS + + protected static class DBUtils { + + + private static Pattern pattern = Pattern.compile("(?!=\\d\\.\\d\\.)([\\d.]+)"); + + public static BBOX parseST_Extent(String extent) throws DataParsingException { + //BOX(11.9122574810083 44.2514144864263,11.9761128271586 44.2912342569845) + try { + log.debug("Parsing BBOX "+extent); + Matcher m=pattern.matcher(extent); + + // Scanner sc = new Scanner(extent); + // double minLong = sc.nextDouble(), + // minLat = sc.nextDouble(), + // maxLong = sc.nextDouble(), + // maxLat= sc.nextDouble(); + + if(!m.find()) throw new DataParsingException("Unable to get minLong "); + Double minLong=Double.parseDouble(m.group(1)); + + if(!m.find()) throw new DataParsingException("Unable to get minLat "); + Double minLat=Double.parseDouble(m.group(1)); + + if(!m.find()) throw new DataParsingException("Unable to get maxLong "); + Double maxLong=Double.parseDouble(m.group(1)); + + if(!m.find()) throw new DataParsingException("Unable to get maxLat "); + Double maxLat=Double.parseDouble(m.group(1)); + return new BBOX(maxLat, maxLong, minLat, minLong); + }catch(Throwable t) { + throw new DataParsingException("Invalid BBOX "+extent,t); + } + } + } +} + diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManagerI.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManagerI.java new file mode 100644 index 0000000..6cb64af --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManagerI.java @@ -0,0 +1,38 @@ +package org.gcube.application.cms.sdi.engine; + +import org.gcube.application.cms.sdi.faults.DataParsingException; +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; + + + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +public interface PostgisDBManagerI { + + void commit() throws SQLException; + + + + + + void create(PostgisTable toCreate) throws SQLException; + PreparedStatement prepareInsertStatement(PostgisTable target, boolean createTable, boolean geometryAsText) + throws SQLException; + void deleteTable(String tableName) throws SQLException; + void truncate(String tableName) throws SQLException; + int deleteByFieldValue(PostgisTable target, PostgisTable.Field field, Object value) throws SQLException; + + + + + // DatabaseConnection getConnectionDescriptor(); + +// PostgisTable.POINT evaluateCentroid(PostgisTable table) throws SQLException, DataParsingException; + + +// ResultSet queryAll(PostgisTable table) throws SQLException; + + +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java new file mode 100644 index 0000000..b23d466 --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java @@ -0,0 +1,105 @@ +package org.gcube.application.cms.sdi.engine; + +import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder; +import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; +import lombok.Data; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.gcube.application.cms.sdi.faults.SDIInteractionException; +import org.gcube.application.geoportal.common.model.profile.Profile; +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; +import org.gcube.application.geoportal.common.utils.Files; + +import java.net.MalformedURLException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + + +@Slf4j +public class PostgisIndexer { + + + public static void init() throws ClassNotFoundException { + + Class.forName("org.postgresql.Driver"); + Class.forName("org.postgis.DriverWrapper"); + + }; + + @NonNull + SDIManagerWrapper manager; + @NonNull + Profile profile; + + @NonNull + DatabaseConnection connectionParameters; + + PostgisDBManagerI dbManager=null; + + public PostgisIndexer(SDIManagerWrapper manager, Profile profile, + DatabaseConnection postgisConnection) throws SQLException { + log.info("POSTGIS Index for {} Connecting to {} ",profile.getId(),postgisConnection); + this.connectionParameters=postgisConnection; + dbManager=new PostgisDBManager(DriverManager. + getConnection(connectionParameters.getUrl(), + connectionParameters.getUser(), + connectionParameters.getPwd())); + this.manager=manager; + this.profile = profile; + + } + + PostgisTable table = null; + + public void initIndex(String indexName, List fields, String workspace,String storeName) throws SQLException, SDIInteractionException { + log.info("Check/init index for {} ",profile.getId()); + table = new PostgisTable(indexName,fields, PostgisTable.GeometryType.POINT); + log.trace("Index Postgis Table is {} ",table); + log.debug("Create if missing.."); + // Check if table exists + dbManager.create(table); + + log.debug("Checking/ registering index layer in GS "); + manager.configureCentroidLayer(indexName,workspace,storeName,table,connectionParameters); + + + // TODO Additional layers + // Create layer + // Publish cross related layers + } + + + + + + + public void insert(Document toInsertRecord)throws SDIInteractionException { + log.info("Inserting {} in index {}",toInsertRecord,table.getTablename()); + try { + PreparedStatement ps = dbManager.prepareInsertStatement(table, false, true); + table.fillObjectsPreparedStatement(toInsertRecord, ps); + ps.execute(); + }catch (Throwable t ){ + log.error("Unable to insert {} into {} ",toInsertRecord,table.getTablename(),t); + throw new SDIInteractionException("Unable to insert record in postgis index "+table.getTablename(),t); + } + } + + public void deleteByStringValue(PostgisTable.Field field, String id) throws SDIInteractionException { + log.info("Deleting {}={} from index {}",field.getName(), id,table.getTablename()); + try { + dbManager.deleteByFieldValue(table,field,id); + }catch (Throwable t ){ + log.error("Unable to delete {}={} from index {}",field.getName(), id,table.getTablename(),t); + throw new SDIInteractionException("Unable to delete record in postgis index "+table.getTablename(),t); + } + } + + + +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java new file mode 100644 index 0000000..e72059d --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java @@ -0,0 +1,316 @@ +package org.gcube.application.cms.sdi.engine; + +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.sdi.faults.DataParsingException; +import org.gcube.application.geoportal.common.model.legacy.BBOX; + + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +@RequiredArgsConstructor +@Getter +@ToString +public class PostgisTable { + + + @Getter + @AllArgsConstructor + public static enum GeometryType{ + MULTIPOINT("4326","geometry (MULTIPOINT,4326)","",""), + POINT("4326","geometry (POINT,4326)","",""), + LINE("4326","geometry (MULTILINESTRING,4326)","",""), + POLYGON("4326","geometry (MULTIPOLYGON,4326)","",""); + private final String SRID; + private final String definition; + private final String InsertWKT; + private final String insertWKB; + + + } + + @RequiredArgsConstructor + @Getter + @Setter + @ToString + public static class Field{ + @NonNull + private String name; + @NonNull + private FieldType type; + private Boolean isIndexed; + private Object constantValue; + } + + @Getter + @AllArgsConstructor + public enum FieldType{ + INT("int",java.sql.Types.INTEGER), + TEXT("text",java.sql.Types.LONGVARCHAR), + FLOAT("float",java.sql.Types.FLOAT), + GEOMETRY("",0), + AUTOINCREMENT("BIGSERIAL PRIMARY KEY",java.sql.Types.BIGINT); + + private String definition; + private int sqlType; + } + + + @RequiredArgsConstructor + @Getter + @ToString + public static class POINT{ + + private static Pattern pattern = Pattern.compile("(?!=\\d\\.\\d\\.)([\\d.]+)"); + + public static POINT parsePOINT(String point) throws DataParsingException { + //POINT(8.30230113965909 44.8011688237011) + // x,y + try { + log.debug("Parsing POINT "+point); + Matcher m=pattern.matcher(point); + + if(!m.find()) throw new DataParsingException("Unable to get x "); + Double x=Double.parseDouble(m.group(1)); + + if(!m.find()) throw new DataParsingException("Unable to get y "); + Double y=Double.parseDouble(m.group(1)); + + return new POINT(x,y); + }catch(Throwable t) { + throw new DataParsingException("Invalid POINT "+point,t); + } + } + + @NonNull + private Double x; + @NonNull + private Double y; + + } + + + private static final NumberFormat DECIMAL_FORMAT=NumberFormat.getInstance(Locale.US); + + static { + ((DecimalFormat) DECIMAL_FORMAT).setGroupingUsed(false); + } + + + public String getGeometryColumn() { + for(Field f:fields) + if(f.getType().equals(FieldType.GEOMETRY)) return f.getName(); + return null; + } + + + @NonNull + private String tablename; + + @NonNull + private List fields; + + + @NonNull + private GeometryType geometryColumnType; + + @Setter + private BBOX boundingBox=null; + + @Setter + private POINT centroid=null; + + public void setTablename(String tablename) { + this.tablename = sanitizeFieldName(tablename); + } + + public String getCreateStatement() { + StringBuilder stmt=new StringBuilder(); + stmt.append("CREATE TABLE IF NOT EXISTS "+tablename+"( "); + for(Field field:fields){ + + String fieldDefinition=field.getType().getDefinition(); + if(field.getType().equals(FieldType.GEOMETRY)) + fieldDefinition=this.getGeometryColumnType().definition; + + stmt.append(field.getName()+" "+fieldDefinition+","); + } + stmt.deleteCharAt(stmt.lastIndexOf(",")); + stmt.append(")"); + return stmt.toString(); + } + + public String getDeleteByFieldStatement(Field field) { + return "DELETE FROM "+tablename+" WHERE "+field.getName()+" = ? "; + } + + + public String getInsertionStatement(boolean geometryText) { + StringBuilder fieldList=new StringBuilder(); + StringBuilder fieldInsertion=new StringBuilder(); + + for(Field field:fields) { + switch(field.getType()) { + case AUTOINCREMENT : break; + case GEOMETRY : { + fieldList.append(field.getName()+","); + if(geometryText) + fieldInsertion.append("ST_GeomFromText(?, 4326),"); + else + fieldInsertion.append("ST_GeomFromWKB(?, 4326),"); + break; + } + default : { + fieldList.append(field.getName()+","); + fieldInsertion.append("?,"); + } + } + } + + fieldList.deleteCharAt(fieldList.lastIndexOf(",")); + fieldInsertion.deleteCharAt(fieldInsertion.lastIndexOf(",")); + + + + return "Insert into "+tablename+" ("+fieldList+") VALUES ("+fieldInsertion+")"; + } + + + public void fillObjectsPreparedStatement(Map row, PreparedStatement toFill) throws SQLException { + int psFieldIndex=0; + HashMap rowValues=new HashMap(); + for(Map.Entry entry:row.entrySet()) + rowValues.put(sanitizeFieldName(entry.getKey()), entry.getValue()); + + + for(Field field:fields) { + if(!field.getType().equals(FieldType.AUTOINCREMENT)) { + psFieldIndex++; + + Object value=rowValues.get(field.getName()); + setObjectInPreparedStatement(field,value,toFill,psFieldIndex); + + } + + + } + } + + + public void setObjectInPreparedStatement(Field field,Object value, PreparedStatement toFill, int psFieldIndex) throws SQLException { + if(value==null) { + try{ + toFill.setNull(psFieldIndex, field.getType().sqlType); + }catch(SQLException e) { + log.error("Unable to set null for field "+field); + throw e; + } + } + else + switch(field.getType()) { + case FLOAT :{ + toFill.setFloat(psFieldIndex, (Float)value); + break; + } + case INT : { + toFill.setInt(psFieldIndex, (Integer)value); + break; + } + case TEXT : { + toFill.setString(psFieldIndex, value.toString()); + break; + } + case GEOMETRY : { + toFill.setBytes(psFieldIndex, (byte[])value); + } + } + } + + public void fillCSVPreparedStatament(Map row, PreparedStatement toFill,boolean explicitGeometry) throws SQLException { + int psFieldIndex=0; + + HashMap rowValues=new HashMap(); + for(Map.Entry entry:row.entrySet()) + rowValues.put(sanitizeFieldName(entry.getKey()), entry.getValue()); + + for(Field field:fields) { + + + if(!field.getType().equals(FieldType.AUTOINCREMENT)) { + psFieldIndex++; + + String value=rowValues.get(field.getName()); + +// if(value==null||value.equalsIgnoreCase("null")) toFill.setNull(psFieldIndex, field.getType().sqlType); +// else + switch(field.getType()) { + case FLOAT :{ + try{ + toFill.setFloat(psFieldIndex, Float.parseFloat(value)); + }catch(NumberFormatException e) { + throw new SQLException(field+" cannot be null. CSV Row is "+rowValues,e); + } + break; + } + case INT : { + try{ + toFill.setInt(psFieldIndex, Integer.parseInt(value)); + }catch(NumberFormatException e) { + log.warn("Skipping value for "+field+" row was "+rowValues,e); + toFill.setNull(psFieldIndex, java.sql.Types.INTEGER); + } + break; + } + case TEXT : { + toFill.setString(psFieldIndex, value.toString()); + break; + } + case GEOMETRY : { + if(explicitGeometry) { + toFill.setString(psFieldIndex,value); + }else { + switch(geometryColumnType){ + case POINT: { + String xRepresentation=DECIMAL_FORMAT.format(Double.parseDouble(rowValues.get(DBConstants.Defaults.XCOORD_FIELD))); + String yRepresentation=DECIMAL_FORMAT.format(Double.parseDouble(rowValues.get(DBConstants.Defaults.YCOORD_FIELD))); + + toFill.setString(psFieldIndex, "POINT("+xRepresentation+" "+ + yRepresentation+")"); + break; + } + default :{ + toFill.setString(psFieldIndex,rowValues.get("wkt")); + break; + } + } + } + break; + } + } + } + + + + + } + } + + public static String sanitizeFieldName(String fieldName) { +// return fieldName.toLowerCase().replaceAll(" ", "_").replaceAll("\\.", "").replaceAll("-", "_").replaceAll("////","_"); + return fieldName.toLowerCase().replaceAll("[^a-z0-9_\\\\]", "_"); + } + + + + +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java new file mode 100644 index 0000000..bab626f --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java @@ -0,0 +1,180 @@ +package org.gcube.application.cms.sdi.engine; + +import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; +import it.geosolutions.geoserver.rest.GeoServerRESTReader; +import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; +import it.geosolutions.geoserver.rest.decoder.RESTLayer; +import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.gcube.application.cms.sdi.faults.SDIInteractionException; +import org.gcube.application.cms.plugins.requests.BaseExecutionRequest; +import org.gcube.application.cms.serialization.Serialization; +import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer; + +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; + +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; +import org.gcube.application.geoportal.common.utils.Files; + + + +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.spatial.data.gis.GISInterface; +import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +public class SDIManager { + + + static protected final String EPSG_4326="EPSG:4326"; + static final String WGS84_FULL="GEOGCS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", SPHEROID[\"WGS 84\", 6378137.0, 298.257223563, AUTHORITY[\"EPSG\",\"7030\"]],"+ + "AUTHORITY[\"EPSG\",\"6326\"]], PRIMEM[\"Greenwich\", 0.0, AUTHORITY[\"EPSG\",\"8901\"]], UNIT[\"degree\", 0.017453292519943295],"+ + "AXIS[\"Geodetic longitude\", EAST], AXIS[\"Geodetic latitude\", NORTH], AUTHORITY[\"EPSG\",\"4326\"]]"; + + + public static final Pattern HOSTNAME_PATTERN=Pattern.compile("(?<=\\:\\/\\/)[^\\:]*"); + public static final Pattern PORT_PATTERN=Pattern.compile("(?<=\\:)[\\d]+"); + public static final Pattern DB_NAME_PATTERN=Pattern.compile("(?<=\\/)[^\\/]*(?=$)"); + + + private final GISInterface gis; + @Getter + private final DataTransferClient dtGeoServer; + @Getter + private final String geoserverHostName; + @Getter + private final AbstractGeoServerDescriptor currentGeoserver; + + + public SDIManager() throws SDIInteractionException { + try{ + log.debug("Initializing GIS Interface.."); + gis=GISInterface.get(); + currentGeoserver=gis.getCurrentGeoServer(); + if(currentGeoserver==null) + throw new Exception("Unable to contact data transfer for geoserver "); + + log.debug("Found geoserver descriptor "+currentGeoserver); + geoserverHostName=new URL(currentGeoserver.getUrl()).getHost(); + + log.debug("Contacting Data Transfer from geoserver {} ",geoserverHostName); + dtGeoServer=DataTransferClient.getInstanceByEndpoint("https://"+geoserverHostName); + if(!currentGeoserver.getReader().existGeoserver()) + throw new Exception("Geoserver not reachable"); + }catch(Exception e) { + throw new SDIInteractionException("Unable to initialize SDI Manager",e); + } + } + + + + + public String createWorkspace(String toCreate) throws SDIInteractionException { + try { + if(!currentGeoserver.getReader().getWorkspaceNames().contains(toCreate)) { + log.debug("Creating workspace : "+toCreate); + if(!currentGeoserver.getPublisher().createWorkspace(toCreate)) + throw new SDIInteractionException("Unable to create workspace "+toCreate); + }else log.debug("Workspace "+toCreate+" exists."); + return toCreate; + } catch (IllegalArgumentException | MalformedURLException e) { + throw new SDIInteractionException("Unable to create workspace "+toCreate,e); + } + } + + + protected static final String getToUseBaseLayerName(RegisteredFileSet fileset){ + // ******** IDENTIFY LAYER NAME correct layer name + // Must be unique under same WS + // equal to shp base name + + String baseName= ""; + + // Chose layer name first identifying filename + for(Object p:fileset.getPayloads()){ + RegisteredFile f = Serialization.convert(p,RegisteredFile.class); + String name=f.getName(); + if(name.endsWith(".shp")) { + log.debug("SHP is {}",name); + baseName= Files.fixFilename(name.substring(0,name.lastIndexOf('.'))); + break; + } + } + return baseName; + } + + protected String createStore(GSPostGISDatastoreEncoder encoder, String workspace) throws SDIInteractionException { + String storeName=encoder.getName(); + try { + log.debug("Looking for datastore "+storeName+" under "+workspace); + + if(currentGeoserver.getReader().getDatastore(workspace,storeName)==null) + + if(!currentGeoserver.getDataStoreManager().create(workspace, encoder)) + throw new SDIInteractionException("Unable to create store "+storeName+" in "+workspace); + log.debug("Store "+storeName+" exists under "+workspace); + return storeName; + } catch (IllegalArgumentException | MalformedURLException e) { + throw new SDIInteractionException("Unable to create store "+storeName,e); + } + } + + + protected String createStoreFromPostgisDB(String workspace, String storeName, DatabaseConnection connection) throws SDIInteractionException{ + String connectionUrl=connection.getUrl(); + + Matcher hostname=HOSTNAME_PATTERN.matcher(connectionUrl); + if (!hostname.find()) throw new SDIInteractionException("Unable to get Hostname from "+connection); + + Matcher port = PORT_PATTERN.matcher(connectionUrl); + if (!port.find()) throw new SDIInteractionException("Unable to get PORT from "+connection); + + Matcher db = DB_NAME_PATTERN.matcher(connectionUrl); + if (!db.find()) throw new SDIInteractionException("Unable to get DB from "+connection); + + + GSPostGISDatastoreEncoder encoder=new GSPostGISDatastoreEncoder(storeName); + encoder.setHost(hostname.group()); + encoder.setPort(Integer.parseInt(port.group())); + encoder.setDatabase(db.group()); + encoder.setSchema("public"); + encoder.setUser(connection.getUser()); + encoder.setPassword(connection.getPwd()); + encoder.setLooseBBox(true); + encoder.setDatabaseType("postgis"); + encoder.setEnabled(true); + encoder.setFetchSize(1000); + encoder.setValidateConnections(true); + + return createStore(encoder,workspace); + } + + protected String publishStyle(File sldFile, String name) throws SDIInteractionException { + try { + if(!currentGeoserver.getReader().existsStyle(name)) { + log.debug("Registering style "+name); + if(!currentGeoserver.getPublisher().publishStyle(sldFile, name)) + throw new SDIInteractionException("Unable to register style "+name); + }else log.debug("Style "+name+" already existing"); + return name; + } catch (IllegalArgumentException | MalformedURLException e) { + throw new SDIInteractionException("Unable to create style "+name,e); + } + + } + +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java new file mode 100644 index 0000000..4d661b7 --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java @@ -0,0 +1,257 @@ +package org.gcube.application.cms.sdi.engine; + +import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; +import it.geosolutions.geoserver.rest.GeoServerRESTReader; +import it.geosolutions.geoserver.rest.decoder.RESTFeatureType; +import it.geosolutions.geoserver.rest.decoder.RESTLayer; +import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder; +import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.gcube.application.cms.caches.AbstractScopedMap; +import org.gcube.application.cms.plugins.requests.BaseExecutionRequest; +import org.gcube.application.cms.sdi.faults.SDIInteractionException; +import org.gcube.application.cms.serialization.Serialization; +import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile; +import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet; +import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; +import org.gcube.application.geoportal.common.utils.Files; +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import sun.reflect.generics.scope.AbstractScope; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class SDIManagerWrapper extends SDIManager{ + + public SDIManagerWrapper() throws SDIInteractionException { + } + + /** + * Expected paramters : + * - "workspace" + * - "layerTitle" + * - "documentID" + * - "basePersistencePath" (profile specific, e.g. "GNA") + * + * @param fileSet + * @param params + * @return + * @throws SDIInteractionException + */ + public RegisteredFileSet materializeLayer(RegisteredFileSet fileSet, Document params) throws SDIInteractionException{ + try { + log.debug("Materializing FS {} on {} ", fileSet, getGeoserverHostName()); + + // validate parameters + String workspace = BaseExecutionRequest.getMandatory("workspace", params); + String documentID = BaseExecutionRequest.getMandatory("documentID", params); + String basePersistencePAth = BaseExecutionRequest.getMandatory("basePersistencePath", params); + + + // check if empty + if (fileSet.getPayloads().isEmpty()) throw new SDIInteractionException("No payload to materialize"); + + + Document geoserverInfo = new Document(); + geoserverInfo.put("_type", "Geoserver"); + geoserverInfo.put("workspace", workspace); + + + // Evaluate Layer Name + String baseName = getToUseBaseLayerName(fileSet); + log.debug("Base layer name is {}, checking conflicts.. "); + String toSetLayerName = baseName; + //Check if layer already exists + int count = 0; + GeoServerRESTReader gsReader = getCurrentGeoserver().getReader(); + while (gsReader.getLayer(workspace, toSetLayerName) != null) { + count++; + toSetLayerName = baseName + "_" + count; + log.debug("layer for " + baseName + " already existing, trying " + toSetLayerName); + } + geoserverInfo.put("layerName", toSetLayerName); + log.debug("Layer name will be {}", toSetLayerName); + + + String folderRelativePath = basePersistencePAth + "/" + documentID + "/" + fileSet.getUUID() + "/" + toSetLayerName; + log.debug("GS Relative destination path is {}", folderRelativePath); + geoserverInfo.put("persistencePath", folderRelativePath); + + List filenames = new ArrayList<>(); + + String absolutePath = null; + + for (Object o : fileSet.getPayloads()) { + RegisteredFile file = Serialization.convert(o, RegisteredFile.class); + log.info("Sending {} to GS {} at {} ", file, getGeoserverHostName(), folderRelativePath); + String completeFilename = Files.fixFilename(file.getName()); + completeFilename = completeFilename.replaceAll(baseName, toSetLayerName); + + + Destination destination = new Destination(completeFilename); + destination.setCreateSubfolders(true); + destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); + destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + + destination.setPersistenceId("geoserver"); + destination.setSubFolder(folderRelativePath); + + log.debug("Sending {} to {}", file, destination); + TransferResult result = getDtGeoServer().httpSource(new URL(file.getLink()), destination); + log.debug("Transferred " + result); + + + filenames.add(completeFilename); + // NB Clash con subfolder is APPEND, thus FOLDER is expected to be the one specified by caller + //geoserverInfo.put(""result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/"))); + absolutePath = result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/")); + } + geoserverInfo.put("files", filenames); + + // Publishing layer in GS + String storeName = toSetLayerName + "_store"; + geoserverInfo.put("storeName", storeName); + + GeoServerRESTPublisher publisher = getCurrentGeoserver().getPublisher(); + log.debug("Trying to create remote workspace : " + workspace); + createWorkspace(workspace); + + + log.debug("Publishing remote folder " + absolutePath); + + URL directoryPath = new URL("file:" + absolutePath + "/" + toSetLayerName + ".shp"); + + //TODO Evaluate SRS + + boolean published = publisher.publishShp( + workspace, + storeName, + null, + toSetLayerName, + // UploadMethod.FILE, // neeeds zip + GeoServerRESTPublisher.UploadMethod.EXTERNAL, // needs shp + directoryPath.toURI(), + EPSG_4326, //SRS + ""); // default style + + if (!published) { + throw new SDIInteractionException("Unable to publish layer " + toSetLayerName + " under " + workspace + ". Unknown Geoserver fault."); + } + + RESTLayer l = gsReader.getLayer(workspace, toSetLayerName); + RESTFeatureType f = gsReader.getFeatureType(l); + + + List ogcLinks = new ArrayList<>(); + + Document wmsLink = new Document(); + wmsLink.put("wms", String.format("https://%1$s/geoserver/%2$s/wms?" + + "service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&" + + "styles=&bbox=%4$f,%5$f,%6$f,%7$f&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d", + getGeoserverHostName(), + workspace, + toSetLayerName, + f.getMinX(), + f.getMinY(), + f.getMaxX(), + f.getMaxY(), + EPSG_4326, + 400, + 400)); + ogcLinks.add(wmsLink); + + List platformInfo = new ArrayList<>(); + platformInfo.add(geoserverInfo); + // TODO Metadata + + + // Materialization object + GCubeSDILayer materialization = new GCubeSDILayer(); + materialization.put(GCubeSDILayer.OGC_LINKS, ogcLinks); + materialization.put(GCubeSDILayer.B_BOX, new GCubeSDILayer.BBOX(f.getMaxX(), f.getMaxY(), f.getMinX(), f.getMinY())); + materialization.put(GCubeSDILayer.PLATFORM_INFO, platformInfo); + + log.info("Generated Materialization {}", materialization); + + //Add Materialization to registered file set + List materializations = fileSet.getMaterializations(); + if (materializations == null) materializations = new ArrayList(); + materializations.add(materialization); + fileSet.put(RegisteredFileSet.MATERIALIZATIONS, materializations); + + return fileSet; + }catch(SDIInteractionException e){ + throw e; + }catch (Throwable t){ + throw new SDIInteractionException("Unexpected exception while trying to materialize File Set "+t.getMessage(),t); + } + } + + + public String configureCentroidLayer(String name, String workspace, String storeName, PostgisTable table, DatabaseConnection connection) throws SDIInteractionException { + + GSFeatureTypeEncoder fte=new GSFeatureTypeEncoder(); + fte.setAbstract("Centroid layer for "+name); + fte.setEnabled(true); + fte.setNativeCRS(WGS84_FULL); + fte.setTitle(name); + fte.setName(name); + + + // GeoServer loads all fields +// fte.setAttribute(attrs); + + + fte.setLatLonBoundingBox(-180.0, -90.0, 180.0, 90.0, WGS84_FULL); + + String style="clustered_centroids"; + + GSLayerEncoder layerEncoder=new GSLayerEncoder(); + layerEncoder.setDefaultStyle(style); + layerEncoder.setEnabled(true); + layerEncoder.setQueryable(true); + try { + //Checking workspace + createWorkspace(workspace); + //Checking store + createStoreFromPostgisDB(workspace, storeName,connection); + //Checking layer + publishStyle(Files.getFileFromResources("styles/clustered_points.sld"),style); + + log.info("Creating layer in {} : {} with FTE {} , LE {}",workspace,storeName,fte,layerEncoder); + if(getCurrentGeoserver().getReader().getLayer(workspace, name)==null) + if(!getCurrentGeoserver().getPublisher().publishDBLayer(workspace, storeName, fte, layerEncoder)) + throw new SDIInteractionException("Unable to create layer "+name); + log.debug("layer "+name+" already exists"); + + + String link=String.format("https://%1$s/geoserver/%2$s/wms?" + +"service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&" + + "styles=&bbox=%4$s,%5$s,%6$s,%7$s&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d", + getGeoserverHostName(), + workspace, + name, + "-1563071.166172796", + "4789738.204048398", + "4334926.486925308", + "5828118.072551585", + EPSG_4326, + 400, + 400); + + return name; + } catch (IllegalArgumentException | MalformedURLException e) { + throw new SDIInteractionException("Unable to create layer "+name,e); + } + + + } +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/faults/DataParsingException.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/faults/DataParsingException.java new file mode 100644 index 0000000..b1f318d --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/faults/DataParsingException.java @@ -0,0 +1,26 @@ +package org.gcube.application.cms.sdi.faults; + +public class DataParsingException extends Exception { + + public DataParsingException() { + } + + public DataParsingException(String message) { + super(message); + } + + public DataParsingException(Throwable cause) { + super(cause); + } + + public DataParsingException(String message, Throwable cause) { + super(message, cause); + } + + public DataParsingException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + + } + +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/faults/SDIInteractionException.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/faults/SDIInteractionException.java similarity index 90% rename from concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/faults/SDIInteractionException.java rename to concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/faults/SDIInteractionException.java index 2c68f7e..5f9d230 100644 --- a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/faults/SDIInteractionException.java +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/faults/SDIInteractionException.java @@ -1,4 +1,4 @@ -package org.gcube.application.cms.concessioni.sdi.faults; +package org.gcube.application.cms.sdi.faults; public class SDIInteractionException extends Exception { diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/model/SDIConfiguration.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/model/SDIConfiguration.java similarity index 84% rename from concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/model/SDIConfiguration.java rename to concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/model/SDIConfiguration.java index 04d196e..3374a56 100644 --- a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/sdi/model/SDIConfiguration.java +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/model/SDIConfiguration.java @@ -1,4 +1,4 @@ -package org.gcube.application.cms.concessioni.sdi.model; +package org.gcube.application.cms.sdi.model; import org.bson.Document; diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java new file mode 100644 index 0000000..28f6899 --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java @@ -0,0 +1,82 @@ +package org.gcube.application.cms.sdi.plugins; + +import com.fasterxml.jackson.databind.cfg.ContextAttributes; +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.caches.AbstractScopedMap; +import org.gcube.application.cms.implementations.ISInterface; +import org.gcube.application.cms.implementations.ImplementationProvider; +import org.gcube.application.cms.plugins.AbstractPlugin; +import org.gcube.application.cms.plugins.InitializablePlugin; +import org.gcube.application.cms.plugins.faults.InitializationException; +import org.gcube.application.cms.plugins.faults.ShutDownException; +import org.gcube.application.cms.plugins.model.PluginDescriptor; +import org.gcube.application.cms.plugins.reports.InitializationReport; +import org.gcube.application.cms.plugins.reports.Report; +import org.gcube.application.cms.sdi.engine.SDIManager; +import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; +import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAmount; +import java.time.temporal.TemporalUnit; + +@Slf4j +public abstract class SDIAbstractPlugin extends AbstractPlugin implements InitializablePlugin { + + protected static AbstractScopedMap sdiCache; + + protected static AbstractScopedMap postgisCache; + + @Synchronized + private static void initCache(){ + if(sdiCache==null) { + log.info("Creating internal caches.. "); + sdiCache = new AbstractScopedMap("SDI-CACHE") { + @Override + protected SDIManagerWrapper retrieveObject() throws ConfigurationException { + try { + return new SDIManagerWrapper(); + } catch (Exception e) { + throw new ConfigurationException(e); + } + } + }; + sdiCache.setTTL(Duration.of(10, ChronoUnit.MINUTES)); + } + if(postgisCache==null) { + postgisCache = new AbstractScopedMap("POSTGIS-CREDENTIALS") { + @Override + protected DatabaseConnection retrieveObject() throws ConfigurationException { + try { + return ImplementationProvider.get().getEngineByClass(ISInterface.class). + queryForDatabase("postgis", "Concessioni"); + } catch (Exception e) { + throw new ConfigurationException(e); + } + } + }; + postgisCache.setTTL(Duration.of(10, ChronoUnit.MINUTES)); + } + } + + + @Override + public InitializationReport init() throws InitializationException { + InitializationReport report = new InitializationReport(); + try{ + initCache(); + report.setStatus(Report.Status.OK); + } catch (Throwable e) { + throw new InitializationException("SDI Plugins : Unable to initialize Internal Caches ",e); + } + return report; + } + + @Override + public void shutdown() throws ShutDownException { + + } +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java new file mode 100644 index 0000000..c2f4a76 --- /dev/null +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java @@ -0,0 +1,214 @@ +package org.gcube.application.cms.sdi.plugins; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.gcube.application.cms.caches.AbstractScopedMap; +import org.gcube.application.cms.implementations.ISInterface; +import org.gcube.application.cms.implementations.ImplementationProvider; +import org.gcube.application.cms.plugins.AbstractPlugin; +import org.gcube.application.cms.plugins.IndexerPluginInterface; +import org.gcube.application.cms.plugins.InitializablePlugin; +import org.gcube.application.cms.plugins.faults.InitializationException; +import org.gcube.application.cms.plugins.faults.ShutDownException; +import org.gcube.application.cms.plugins.model.PluginDescriptor; +import org.gcube.application.cms.plugins.reports.IndexDocumentReport; +import org.gcube.application.cms.plugins.reports.InitializationReport; +import org.gcube.application.cms.plugins.reports.MaterializationReport; +import org.gcube.application.cms.plugins.reports.Report; +import org.gcube.application.cms.plugins.requests.BaseExecutionRequest; +import org.gcube.application.cms.plugins.requests.IndexDocumentRequest; +import org.gcube.application.cms.sdi.engine.PostgisIndexer; +import org.gcube.application.cms.sdi.engine.PostgisTable; +import org.gcube.application.cms.sdi.engine.SDIManager; +import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; +import org.gcube.application.cms.sdi.faults.SDIInteractionException; +import org.gcube.application.cms.serialization.Serialization; +import org.gcube.application.geoportal.common.model.JSONPathWrapper; +import org.gcube.application.geoportal.common.model.document.ComparableVersion; +import org.gcube.application.geoportal.common.model.document.ProfiledDocument; +import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer; +import org.gcube.application.geoportal.common.model.profile.HandlerDeclaration; +import org.gcube.application.geoportal.common.model.profile.Profile; +import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; +import org.gcube.application.geoportal.common.utils.ContextUtils; + +import javax.swing.border.AbstractBorder; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPluginInterface { + + @Data + private class MappingObject{ + private String name; + private String type; + private String path; + } + + + + private static final PluginDescriptor DESCRIPTOR=new PluginDescriptor("SDI-Indexer-Plugin", + PluginDescriptor.BaseTypes.INDEXER); + + static { + DESCRIPTOR.setDescription("SDI Indexer. " + + "Manage Centroids layers."); + DESCRIPTOR.setVersion(new ComparableVersion("1.0.0")); + } + + @Override + public PluginDescriptor getDescriptor() { + return DESCRIPTOR; + } + + + @Override + public InitializationReport initInContext() throws InitializationException { + InitializationReport report = new InitializationReport(); + report.setStatus(Report.Status.OK); + return report; + } + + /** + * Expected parameters : + * - indexName (unique) + * - workspace + * + * @param request + * @return + */ + + @Override + public IndexDocumentReport index(IndexDocumentRequest request) { + + log.info("Indxer {} : Performing {} ",this.getDescriptor().getId(),request); + + ProfiledDocument profiledDocument=request.getDocument(); + Profile profile = request.getProfile(); + Document requestArguments=request.getCallParameters(); + Document profileConfiguration =getConfigurationFromProfile(profile).getConfiguration(); + + IndexDocumentReport report= new IndexDocumentReport(); + + log.debug("Profile Configuration is {} ",profileConfiguration); + + try{ + // ********* INIT INDEX + // TODO CACHE + PostgisIndexer indexer = new PostgisIndexer(sdiCache.getObject(),profile,postgisCache.getObject()); + + + // SCHEMA + log.debug("Profile {} : Evaluating Index schema.. ",profile.getId()); + List fields = null; // TODO From Profile + fields.add(new PostgisTable.Field("geom", PostgisTable.FieldType.GEOMETRY)); + fields.add(new PostgisTable.Field("projectid", PostgisTable.FieldType.TEXT)); + List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class); + if(mappingObjs!=null){ + mappingObjs.forEach(o -> { + log.trace("Mapping is {} ",o); + MappingObject m = Serialization.convert(o,MappingObject.class); + fields.add(new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType()))); + }); + } + + + + indexer.initIndex(requestArguments.getString("indexName"), + fields, + requestArguments.getString("workspace"), + requestArguments.getString("indexName")); + + + + // ************* PREPARE RECORD + + + JSONPathWrapper documentNavigator=new JSONPathWrapper(Serialization.write(profiledDocument)); + + Document doc = requestArguments; + // DEFAULT VALUES + doc.put("projectid",profiledDocument.get_id()); + + // ********************** EVALAUTE POSITION + log.debug("indexing Profile {} : Evaluating Centroid... ",profile.getId()); + if(profiledDocument.getSpatialReference()!=null){ + log.debug("Using user defined spatial reference "+profiledDocument.getSpatialReference()); + //TODO USE GEOJSON Position + throw new Exception("Not yet implemented"); + }else { + log.debug("Profile {} : Getting evaluation paths from profile.. ",profile.getId()); + List bboxEvaluationPaths = profileConfiguration.get("bboxEvaluation",List.class); + if(bboxEvaluationPaths==null || bboxEvaluationPaths.isEmpty()) + throw new Exception("Missing configuration bboxEvaluation"); + + GCubeSDILayer.BBOX toSet = null; + for(Object pathObj : bboxEvaluationPaths){ + log.debug("Profile {} : Evaluating path {} ",profile.getId(),pathObj); + for(String path : documentNavigator.getMatchingPaths(pathObj.toString())) { + Object bboxObject = documentNavigator.getByPath(path); + log.info("Matched path {}, value is {} ",path,bboxObject); + GCubeSDILayer.BBOX box = Serialization.convert(bboxObject, GCubeSDILayer.BBOX.class); + + if(toSet == null) toSet = box; + if(box.getMaxX()>toSet.getMaxX()) toSet.setMaxX(box.getMaxX()); + if(box.getMaxY()>toSet.getMaxY()) toSet.setMaxY(box.getMaxY()); + + if(box.getMinX() { + log.trace("Mapping is {} ",o); + MappingObject m = Serialization.convert(o,MappingObject.class); + doc.put(m.getName(),documentNavigator.getByPath(m.getPath()).get(0)); + }); + } + + + indexer.insert(doc); + + String finalDocument = documentNavigator.getValueCTX().jsonString(); + log.debug("Final document after indexing is {} ",finalDocument); + report.setResultingDocument(Document.parse(finalDocument)); + report.setStatus(Report.Status.OK); + }catch (SDIInteractionException e){ + log.error("Unable to index "+request,e); + report.setStatus(Report.Status.ERROR); + report.putMessage(e.getMessage()); + }catch (Throwable t){ + log.error("Unable to index "+request,t); + report.setStatus(Report.Status.ERROR); + report.putMessage(t.getMessage()); + }finally{ + return report; + } + } + +} diff --git a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/SDIMaterializerPlugin.java b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java similarity index 88% rename from concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/SDIMaterializerPlugin.java rename to concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java index 363c73f..ca360b8 100644 --- a/concessioni-lifecycle/src/main/java/org/gcube/application/cms/concessioni/plugins/SDIMaterializerPlugin.java +++ b/concessioni-lifecycle/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java @@ -1,12 +1,13 @@ -package org.gcube.application.cms.concessioni.plugins; +package org.gcube.application.cms.sdi.plugins; import lombok.Data; -import lombok.Getter; import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; import org.bson.Document; -import org.gcube.application.cms.concessioni.sdi.SDIManager; -import org.gcube.application.cms.concessioni.sdi.faults.SDIInteractionException; +import org.gcube.application.cms.plugins.AbstractPlugin; +import org.gcube.application.cms.sdi.engine.SDIManager; +import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; +import org.gcube.application.cms.sdi.faults.SDIInteractionException; import org.gcube.application.cms.plugins.MaterializationPlugin; import org.gcube.application.cms.plugins.faults.InitializationException; import org.gcube.application.cms.plugins.faults.MaterializationException; @@ -31,7 +32,7 @@ import java.util.List; import java.util.Map; @Slf4j -public class SDIMaterializerPlugin implements MaterializationPlugin { +public class SDIMaterializerPlugin extends AbstractPlugin implements MaterializationPlugin { @Data public static class MaterializationConfig{ @@ -39,10 +40,10 @@ public class SDIMaterializerPlugin implements MaterializationPlugin { private String documentPath; } - Map sdiManagerMap=null; + Map sdiManagerMap=null; - private SDIManager getSDIManager(){ + private SDIManagerWrapper getSDIManager(){ return sdiManagerMap.get(ContextUtils.getCurrentScope()); } @@ -54,7 +55,7 @@ public class SDIMaterializerPlugin implements MaterializationPlugin { String context = ContextUtils.getCurrentScope(); if(getSDIManager()==null) { log.info("Initializing in " + context); - sdiManagerMap.put(context,new SDIManager()); + sdiManagerMap.put(context,new SDIManagerWrapper()); } report.setStatus(Report.Status.OK); report.putMessage("Initialized SDI Manager in "+context+" pointing to "+getSDIManager().getGeoserverHostName()); @@ -93,12 +94,7 @@ public class SDIMaterializerPlugin implements MaterializationPlugin { ProfiledDocument profiledDocument=request.getDocument(); Profile profile = request.getProfile(); Document requestArguments=request.getCallParameters(); - Document profileConfiguration =null; - for (HandlerDeclaration h : profile.getHandlers()) { - if (h.getId().equals("SDI-Default-Materializer")) { - profileConfiguration = h.getConfiguration(); - } - } + Document profileConfiguration =getConfigurationFromProfile(profile).getConfiguration(); MaterializationReport report= new MaterializationReport(); @@ -141,8 +137,9 @@ public class SDIMaterializerPlugin implements MaterializationPlugin { } } - - report.setResultingDocument(Document.parse(documentNavigator.getValueCTX().jsonString())); + String finalDocument = documentNavigator.getValueCTX().jsonString(); + log.debug("Final document after materialization is {} ",finalDocument); + report.setResultingDocument(Document.parse(finalDocument)); report.setStatus(Report.Status.OK); }catch (SDIInteractionException e){ log.error("Unable to materialize "+request,e);