From aa5717b78af64698e44b4069d42d302f1b8cab35 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Fri, 4 Mar 2022 11:30:31 +0100 Subject: [PATCH] Test indexer --- sdi-plugins/pom.xml | 7 + .../cms/sdi/engine/PostgisTable.java | 4 +- .../cms/sdi/plugins/SDIAbstractPlugin.java | 5 +- .../cms/sdi/plugins/SDIIndexerPlugin.java | 75 +++++--- .../sdi/plugins/SDIMaterializerPlugin.java | 3 +- .../resources/styles/clustered_points.sld | 173 ++++++++++++++++++ .../cms/sdi/plugins/IndexerTest.java | 8 +- 7 files changed, 240 insertions(+), 35 deletions(-) create mode 100644 sdi-plugins/src/main/resources/styles/clustered_points.sld diff --git a/sdi-plugins/pom.xml b/sdi-plugins/pom.xml index 1c2e5f4..4eda049 100644 --- a/sdi-plugins/pom.xml +++ b/sdi-plugins/pom.xml @@ -49,6 +49,13 @@ [1.2.1,2.0.0] + + + net.postgis + postgis-jdbc + 2.5.0 + + org.gcube.application.cms cms-test-commons diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java index e72059d..d20c121 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java @@ -231,7 +231,9 @@ public class PostgisTable { break; } case GEOMETRY : { - toFill.setBytes(psFieldIndex, (byte[])value); + if(value instanceof String) + toFill.setString(psFieldIndex, ((String) value)); + else toFill.setBytes(psFieldIndex, (byte[])value); } } } diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java index 467eba8..597fa3a 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java @@ -17,6 +17,7 @@ import org.gcube.application.cms.sdi.engine.SDIManager; import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; import org.gcube.application.geoportal.common.model.rest.ConfigurationException; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; +import org.gcube.application.geoportal.common.utils.ContextUtils; import org.gcube.data.transfer.model.ServiceConstants; import java.time.Duration; @@ -52,8 +53,10 @@ public abstract class SDIAbstractPlugin extends AbstractPlugin implements Initia @Override protected DatabaseConnection retrieveObject() throws ConfigurationException { try { - return ImplementationProvider.get().getEngineByClass(ISInterface.class). + DatabaseConnection db = ImplementationProvider.get().getEngineByClass(ISInterface.class). queryForDatabase("Database","postgis", "GNA_DB","Concessioni"); + log.debug("Postgis Connection in {} is {} ", ContextUtils.getCurrentScope(),db); + return db; } catch (Exception e) { throw new ConfigurationException(e); } diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java index 4e7e4c7..451242c 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java @@ -4,7 +4,9 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.gcube.application.cms.plugins.IndexerPluginInterface; +import org.gcube.application.cms.plugins.faults.IndexingException; import org.gcube.application.cms.plugins.faults.InitializationException; +import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException; import org.gcube.application.cms.plugins.faults.InvalidProfileException; import org.gcube.application.cms.plugins.model.PluginDescriptor; import org.gcube.application.cms.plugins.reports.IndexDocumentReport; @@ -19,6 +21,7 @@ import org.gcube.application.geoportal.common.model.JSONPathWrapper; import org.gcube.application.cms.plugins.model.ComparableVersion; import org.gcube.application.geoportal.common.model.document.ProfiledDocument; import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer; +import org.gcube.application.geoportal.common.model.document.spatial.SpatialReference; import org.gcube.application.geoportal.common.model.profile.Profile; import java.util.ArrayList; @@ -33,10 +36,10 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin private String type; private String path; - public void validate () throws RuntimeException { - if(name==null) throw new RuntimeException("Invalid mapping "+this+" : name is null"); - if(type==null) throw new RuntimeException("Invalid mapping "+this+" : type is null"); - if(path==null) throw new RuntimeException("Invalid mapping "+this+" : path is null"); + public void validate () throws InvalidProfileException { + if(name==null) throw new InvalidProfileException("Invalid mapping "+this+" : name is null"); + if(type==null) throw new InvalidProfileException("Invalid mapping "+this+" : type is null"); + if(path==null) throw new InvalidProfileException("Invalid mapping "+this+" : path is null"); } } @@ -68,13 +71,14 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin * Expected parameters : * - indexName (unique) * - workspace + * - centroidRecord (OPT) * * @param request * @return */ @Override - public IndexDocumentReport index(IndexDocumentRequest request) { + public IndexDocumentReport index(IndexDocumentRequest request) throws InvalidPluginRequestException { log.info("Indexer {} : Performing {} ",this.getDescriptor().getId(),request); @@ -100,17 +104,23 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin fields.add(new PostgisTable.Field("geom", PostgisTable.FieldType.GEOMETRY)); fields.add(new PostgisTable.Field("projectid", PostgisTable.FieldType.TEXT)); List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class); + + log.trace("Loading mappings from profile.. "); + List mappingObjects= new ArrayList<>(); if(mappingObjs!=null){ - mappingObjs.forEach(o -> { - log.trace("Mapping is {} ",o); - MappingObject m = Serialization.convert(o,MappingObject.class); + for (Object mappingObj : mappingObjs) { + log.trace("Mapping is {} ",mappingObj); + MappingObject m = Serialization.convert(mappingObj,MappingObject.class); m.validate(); + fields.add(new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType()))); - }); + } } + + indexer.initIndex(requestArguments.getString("indexName"), fields, requestArguments.getString("workspace"), @@ -123,9 +133,11 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin JSONPathWrapper documentNavigator=new JSONPathWrapper(Serialization.write(profiledDocument)); - Document doc = requestArguments; + Document centroidDoc = new Document(); + if(requestArguments.containsKey("centroidRecord")) + centroidDoc.putAll(requestArguments.get("centroidRecords",Document.class)); // DEFAULT VALUES - doc.put("projectid",profiledDocument.getId()); + centroidDoc.put("projectid",profiledDocument.getId()); // ********************** EVALAUTE POSITION log.debug("indexing Profile {} : Evaluating Centroid... ",profile.getId()); @@ -142,9 +154,10 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin GCubeSDILayer.BBOX toSet = null; for(Object pathObj : bboxEvaluationPaths){ log.debug("Profile {} : Evaluating path {} ",profile.getId(),pathObj); - for(String path : documentNavigator.getMatchingPaths(pathObj.toString())) { - Object bboxObject = documentNavigator.getByPath(path).get(0); - log.info("Matched path {}, value is {} ",path,bboxObject); + List bboxObjects = documentNavigator.getByPath(pathObj.toString()); + log.debug("Profile {} : Evaluating path {} .. results {} ",profile.getId(),pathObj,bboxObjects); + for(Object bboxObject : bboxObjects) { + log.info("Matched path {}, value is {} ",pathObj.toString(),bboxObject); GCubeSDILayer.BBOX box = Serialization.convert(bboxObject, GCubeSDILayer.BBOX.class); if(toSet == null) toSet = box; @@ -156,16 +169,19 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin } } + if(toSet == null) + throw new IndexingException("No BBOX has been found on paths : "+bboxEvaluationPaths); + log.info("Evaluated BBOX {} ",toSet); - String wkt = String .format("POINT (%1$d %2$d) ", - toSet.getMaxX()-toSet.getMinX(), + String wkt = String .format("POINT (%1$f %2$f) ", + (toSet.getMaxX()-toSet.getMinX()), toSet.getMaxY()-toSet.getMinY()); // TODO SET Spatial reference - doc.put("geom",wkt); + centroidDoc.put("geom",wkt); } @@ -173,21 +189,22 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin //*********** Additional Values from profile - log.info("Setting additional values"); - if(mappingObjs!=null){ - mappingObjs.forEach(o -> { - log.trace("Mapping is {} ",o); - MappingObject m = Serialization.convert(o,MappingObject.class); - doc.put(m.getName(),documentNavigator.getByPath(m.getPath()).get(0)); - }); + log.info("Setting additional values to centroid from mappings .."); + for(MappingObject m : mappingObjects){ + List foundValues = documentNavigator.getByPath(m.getPath()); + Object toSetValue=null; + if(!foundValues.isEmpty()) + toSetValue=foundValues.get(0); + log.trace("Setting {} = {} in centroid doc ",m.getName(),toSetValue); + centroidDoc.put(m.getName(),foundValues.get(0)); + } + log.info("Inserting Centroid {} into {} ",centroidDoc.toJson(),indexer); + indexer.insert(centroidDoc); - indexer.insert(doc); - - String finalDocument = documentNavigator.getValueCTX().jsonString(); - log.debug("Final document after indexing is {} ",finalDocument); - report.setResultingDocument(Document.parse(finalDocument)); + // TODO GEOJSON + report.setToSetSpatialReference(new SpatialReference()); report.setStatus(Report.Status.OK); }catch (SDIInteractionException e){ log.error("Unable to index "+request,e); diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java index 9464421..1ab4399 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java @@ -5,6 +5,7 @@ import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.gcube.application.cms.plugins.AbstractPlugin; +import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException; import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; import org.gcube.application.cms.sdi.faults.SDIInteractionException; import org.gcube.application.cms.plugins.MaterializationPlugin; @@ -87,7 +88,7 @@ public class SDIMaterializerPlugin extends AbstractPlugin implements Materializa * @throws MaterializationException */ @Override - public MaterializationReport materialize(MaterializationRequest request) throws MaterializationException { + public MaterializationReport materialize(MaterializationRequest request) throws MaterializationException, InvalidPluginRequestException { log.info("Materializer {} : Performing {} ",this.getDescriptor().getId(),request); ProfiledDocument profiledDocument=request.getDocument(); Profile profile = request.getProfile(); diff --git a/sdi-plugins/src/main/resources/styles/clustered_points.sld b/sdi-plugins/src/main/resources/styles/clustered_points.sld new file mode 100644 index 0000000..922502d --- /dev/null +++ b/sdi-plugins/src/main/resources/styles/clustered_points.sld @@ -0,0 +1,173 @@ + + + + vol_stacked_point + + + Stacked Point + Styles archeomar using stacked points + + + + + data + + + cellSize + 30 + + + outputBBOX + + wms_bbox + + + + outputWidth + + wms_width + + + + outputHeight + + wms_height + + + + + + rule1 + Site + + + count + 1 + + + + + + circle + + #FF0000 + + + 8 + + + + + rule29 + 2-9 Sites + + + count + + 2 + + + 9 + + + + + + + circle + + #AA0000 + + + 14 + + + + + + Arial + 12 + bold + + + + + 0.5 + 0.8 + + + + + 2 + + #AA0000 + 0.9 + + + + #FFFFFF + 1.0 + + + + + rule10 + 10 Sites + + + count + 9 + + + + + + circle + + #AA0000 + + + 22 + + + + + + Arial + 12 + bold + + + + + 0.5 + 0.8 + + + + + 2 + + #AA0000 + 0.9 + + + + #FFFFFF + 1.0 + + + + + + + diff --git a/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/IndexerTest.java b/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/IndexerTest.java index 4a24e8d..e5bb512 100644 --- a/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/IndexerTest.java +++ b/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/IndexerTest.java @@ -17,6 +17,7 @@ import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.tests.GCubeTest; import org.junit.Test; +import static junit.framework.TestCase.assertTrue; import static org.junit.Assume.assumeTrue; public class IndexerTest extends BasicPluginTest { @@ -37,11 +38,12 @@ public class IndexerTest extends BasicPluginTest { IndexerPluginInterface plugin = (IndexerPluginInterface) plugins.get(SDIIndexerPlugin.DESCRIPTOR.getId()); IndexDocumentReport response = plugin.index(request); - assumeTrue(response!=null); + assertTrue(response!=null); response.validate(); + System.out.println("Response is "+Serialization.write(response)); - assumeTrue(response.getStatus().equals(Report.Status.OK)); - assumeTrue(response.prepareResult().getSpatialReference()!=null); + assertTrue(response.getStatus().equals(Report.Status.OK)); + assertTrue(response.prepareResult().getSpatialReference()!=null); } }