Test indexer

This commit is contained in:
Fabio Sinibaldi 2022-03-04 11:30:31 +01:00
parent bb1bda69f8
commit aa5717b78a
7 changed files with 240 additions and 35 deletions

View File

@ -49,6 +49,13 @@
<version>[1.2.1,2.0.0]</version> <version>[1.2.1,2.0.0]</version>
</dependency> </dependency>
<!-- POSTGRES DRIVERS -->
<dependency>
<groupId>net.postgis</groupId>
<artifactId>postgis-jdbc</artifactId>
<version>2.5.0</version>
</dependency>
<dependency> <dependency>
<groupId>org.gcube.application.cms</groupId> <groupId>org.gcube.application.cms</groupId>
<artifactId>cms-test-commons</artifactId> <artifactId>cms-test-commons</artifactId>

View File

@ -231,7 +231,9 @@ public class PostgisTable {
break; break;
} }
case GEOMETRY : { case GEOMETRY : {
toFill.setBytes(psFieldIndex, (byte[])value); if(value instanceof String)
toFill.setString(psFieldIndex, ((String) value));
else toFill.setBytes(psFieldIndex, (byte[])value);
} }
} }
} }

View File

@ -17,6 +17,7 @@ import org.gcube.application.cms.sdi.engine.SDIManager;
import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; import org.gcube.application.cms.sdi.engine.SDIManagerWrapper;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException; import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.application.geoportal.common.utils.ContextUtils;
import org.gcube.data.transfer.model.ServiceConstants; import org.gcube.data.transfer.model.ServiceConstants;
import java.time.Duration; import java.time.Duration;
@ -52,8 +53,10 @@ public abstract class SDIAbstractPlugin extends AbstractPlugin implements Initia
@Override @Override
protected DatabaseConnection retrieveObject() throws ConfigurationException { protected DatabaseConnection retrieveObject() throws ConfigurationException {
try { try {
return ImplementationProvider.get().getEngineByClass(ISInterface.class). DatabaseConnection db = ImplementationProvider.get().getEngineByClass(ISInterface.class).
queryForDatabase("Database","postgis", "GNA_DB","Concessioni"); queryForDatabase("Database","postgis", "GNA_DB","Concessioni");
log.debug("Postgis Connection in {} is {} ", ContextUtils.getCurrentScope(),db);
return db;
} catch (Exception e) { } catch (Exception e) {
throw new ConfigurationException(e); throw new ConfigurationException(e);
} }

View File

@ -4,7 +4,9 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.bson.Document; import org.bson.Document;
import org.gcube.application.cms.plugins.IndexerPluginInterface; import org.gcube.application.cms.plugins.IndexerPluginInterface;
import org.gcube.application.cms.plugins.faults.IndexingException;
import org.gcube.application.cms.plugins.faults.InitializationException; import org.gcube.application.cms.plugins.faults.InitializationException;
import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException;
import org.gcube.application.cms.plugins.faults.InvalidProfileException; import org.gcube.application.cms.plugins.faults.InvalidProfileException;
import org.gcube.application.cms.plugins.model.PluginDescriptor; import org.gcube.application.cms.plugins.model.PluginDescriptor;
import org.gcube.application.cms.plugins.reports.IndexDocumentReport; import org.gcube.application.cms.plugins.reports.IndexDocumentReport;
@ -19,6 +21,7 @@ import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.cms.plugins.model.ComparableVersion; import org.gcube.application.cms.plugins.model.ComparableVersion;
import org.gcube.application.geoportal.common.model.document.ProfiledDocument; import org.gcube.application.geoportal.common.model.document.ProfiledDocument;
import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer; import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.spatial.SpatialReference;
import org.gcube.application.geoportal.common.model.profile.Profile; import org.gcube.application.geoportal.common.model.profile.Profile;
import java.util.ArrayList; import java.util.ArrayList;
@ -33,10 +36,10 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
private String type; private String type;
private String path; private String path;
public void validate () throws RuntimeException { public void validate () throws InvalidProfileException {
if(name==null) throw new RuntimeException("Invalid mapping "+this+" : name is null"); if(name==null) throw new InvalidProfileException("Invalid mapping "+this+" : name is null");
if(type==null) throw new RuntimeException("Invalid mapping "+this+" : type is null"); if(type==null) throw new InvalidProfileException("Invalid mapping "+this+" : type is null");
if(path==null) throw new RuntimeException("Invalid mapping "+this+" : path is null"); if(path==null) throw new InvalidProfileException("Invalid mapping "+this+" : path is null");
} }
} }
@ -68,13 +71,14 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
* Expected parameters : * Expected parameters :
* - indexName (unique) * - indexName (unique)
* - workspace * - workspace
* - centroidRecord (OPT)
* *
* @param request * @param request
* @return * @return
*/ */
@Override @Override
public IndexDocumentReport index(IndexDocumentRequest request) { public IndexDocumentReport index(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Performing {} ",this.getDescriptor().getId(),request); log.info("Indexer {} : Performing {} ",this.getDescriptor().getId(),request);
@ -100,14 +104,20 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
fields.add(new PostgisTable.Field("geom", PostgisTable.FieldType.GEOMETRY)); fields.add(new PostgisTable.Field("geom", PostgisTable.FieldType.GEOMETRY));
fields.add(new PostgisTable.Field("projectid", PostgisTable.FieldType.TEXT)); fields.add(new PostgisTable.Field("projectid", PostgisTable.FieldType.TEXT));
List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class); List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class);
log.trace("Loading mappings from profile.. ");
List<MappingObject> mappingObjects= new ArrayList<>();
if(mappingObjs!=null){ if(mappingObjs!=null){
mappingObjs.forEach(o -> { for (Object mappingObj : mappingObjs) {
log.trace("Mapping is {} ",o); log.trace("Mapping is {} ",mappingObj);
MappingObject m = Serialization.convert(o,MappingObject.class); MappingObject m = Serialization.convert(mappingObj,MappingObject.class);
m.validate(); m.validate();
fields.add(new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType()))); fields.add(new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType())));
});
} }
}
@ -123,9 +133,11 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
JSONPathWrapper documentNavigator=new JSONPathWrapper(Serialization.write(profiledDocument)); JSONPathWrapper documentNavigator=new JSONPathWrapper(Serialization.write(profiledDocument));
Document doc = requestArguments; Document centroidDoc = new Document();
if(requestArguments.containsKey("centroidRecord"))
centroidDoc.putAll(requestArguments.get("centroidRecords",Document.class));
// DEFAULT VALUES // DEFAULT VALUES
doc.put("projectid",profiledDocument.getId()); centroidDoc.put("projectid",profiledDocument.getId());
// ********************** EVALAUTE POSITION // ********************** EVALAUTE POSITION
log.debug("indexing Profile {} : Evaluating Centroid... ",profile.getId()); log.debug("indexing Profile {} : Evaluating Centroid... ",profile.getId());
@ -142,9 +154,10 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
GCubeSDILayer.BBOX toSet = null; GCubeSDILayer.BBOX toSet = null;
for(Object pathObj : bboxEvaluationPaths){ for(Object pathObj : bboxEvaluationPaths){
log.debug("Profile {} : Evaluating path {} ",profile.getId(),pathObj); log.debug("Profile {} : Evaluating path {} ",profile.getId(),pathObj);
for(String path : documentNavigator.getMatchingPaths(pathObj.toString())) { List<Object> bboxObjects = documentNavigator.getByPath(pathObj.toString());
Object bboxObject = documentNavigator.getByPath(path).get(0); log.debug("Profile {} : Evaluating path {} .. results {} ",profile.getId(),pathObj,bboxObjects);
log.info("Matched path {}, value is {} ",path,bboxObject); for(Object bboxObject : bboxObjects) {
log.info("Matched path {}, value is {} ",pathObj.toString(),bboxObject);
GCubeSDILayer.BBOX box = Serialization.convert(bboxObject, GCubeSDILayer.BBOX.class); GCubeSDILayer.BBOX box = Serialization.convert(bboxObject, GCubeSDILayer.BBOX.class);
if(toSet == null) toSet = box; if(toSet == null) toSet = box;
@ -156,16 +169,19 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
} }
} }
if(toSet == null)
throw new IndexingException("No BBOX has been found on paths : "+bboxEvaluationPaths);
log.info("Evaluated BBOX {} ",toSet); log.info("Evaluated BBOX {} ",toSet);
String wkt = String .format("POINT (%1$d %2$d) ", String wkt = String .format("POINT (%1$f %2$f) ",
toSet.getMaxX()-toSet.getMinX(), (toSet.getMaxX()-toSet.getMinX()),
toSet.getMaxY()-toSet.getMinY()); toSet.getMaxY()-toSet.getMinY());
// TODO SET Spatial reference // TODO SET Spatial reference
doc.put("geom",wkt); centroidDoc.put("geom",wkt);
} }
@ -173,21 +189,22 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
//*********** Additional Values from profile //*********** Additional Values from profile
log.info("Setting additional values"); log.info("Setting additional values to centroid from mappings ..");
if(mappingObjs!=null){ for(MappingObject m : mappingObjects){
mappingObjs.forEach(o -> { List<Object> foundValues = documentNavigator.getByPath(m.getPath());
log.trace("Mapping is {} ",o); Object toSetValue=null;
MappingObject m = Serialization.convert(o,MappingObject.class); if(!foundValues.isEmpty())
doc.put(m.getName(),documentNavigator.getByPath(m.getPath()).get(0)); toSetValue=foundValues.get(0);
}); log.trace("Setting {} = {} in centroid doc ",m.getName(),toSetValue);
centroidDoc.put(m.getName(),foundValues.get(0));
} }
log.info("Inserting Centroid {} into {} ",centroidDoc.toJson(),indexer);
indexer.insert(centroidDoc);
indexer.insert(doc); // TODO GEOJSON
report.setToSetSpatialReference(new SpatialReference());
String finalDocument = documentNavigator.getValueCTX().jsonString();
log.debug("Final document after indexing is {} ",finalDocument);
report.setResultingDocument(Document.parse(finalDocument));
report.setStatus(Report.Status.OK); report.setStatus(Report.Status.OK);
}catch (SDIInteractionException e){ }catch (SDIInteractionException e){
log.error("Unable to index "+request,e); log.error("Unable to index "+request,e);

View File

@ -5,6 +5,7 @@ import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.bson.Document; import org.bson.Document;
import org.gcube.application.cms.plugins.AbstractPlugin; import org.gcube.application.cms.plugins.AbstractPlugin;
import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException;
import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; import org.gcube.application.cms.sdi.engine.SDIManagerWrapper;
import org.gcube.application.cms.sdi.faults.SDIInteractionException; import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.plugins.MaterializationPlugin; import org.gcube.application.cms.plugins.MaterializationPlugin;
@ -87,7 +88,7 @@ public class SDIMaterializerPlugin extends AbstractPlugin implements Materializa
* @throws MaterializationException * @throws MaterializationException
*/ */
@Override @Override
public MaterializationReport materialize(MaterializationRequest request) throws MaterializationException { public MaterializationReport materialize(MaterializationRequest request) throws MaterializationException, InvalidPluginRequestException {
log.info("Materializer {} : Performing {} ",this.getDescriptor().getId(),request); log.info("Materializer {} : Performing {} ",this.getDescriptor().getId(),request);
ProfiledDocument profiledDocument=request.getDocument(); ProfiledDocument profiledDocument=request.getDocument();
Profile profile = request.getProfile(); Profile profile = request.getProfile();

View File

@ -0,0 +1,173 @@
<?xml version="1.0" encoding="ISO-8859-1"?>
<StyledLayerDescriptor version="1.0.0"
xsi:schemaLocation="http://www.opengis.net/sld StyledLayerDescriptor.xsd"
xmlns="http://www.opengis.net/sld"
xmlns:ogc="http://www.opengis.net/ogc"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<NamedLayer>
<Name>vol_stacked_point</Name>
<UserStyle>
<!-- Styles can have names, titles and abstracts -->
<Title>Stacked Point</Title>
<Abstract>Styles archeomar using stacked points</Abstract>
<FeatureTypeStyle>
<Transformation>
<ogc:Function name="vec:PointStacker">
<ogc:Function name="parameter">
<ogc:Literal>data</ogc:Literal>
</ogc:Function>
<ogc:Function name="parameter">
<ogc:Literal>cellSize</ogc:Literal>
<ogc:Literal>30</ogc:Literal>
</ogc:Function>
<ogc:Function name="parameter">
<ogc:Literal>outputBBOX</ogc:Literal>
<ogc:Function name="env">
<ogc:Literal>wms_bbox</ogc:Literal>
</ogc:Function>
</ogc:Function>
<ogc:Function name="parameter">
<ogc:Literal>outputWidth</ogc:Literal>
<ogc:Function name="env">
<ogc:Literal>wms_width</ogc:Literal>
</ogc:Function>
</ogc:Function>
<ogc:Function name="parameter">
<ogc:Literal>outputHeight</ogc:Literal>
<ogc:Function name="env">
<ogc:Literal>wms_height</ogc:Literal>
</ogc:Function>
</ogc:Function>
</ogc:Function>
</Transformation>
<Rule>
<Name>rule1</Name>
<Title>Site</Title>
<ogc:Filter>
<ogc:PropertyIsLessThanOrEqualTo>
<ogc:PropertyName>count</ogc:PropertyName>
<ogc:Literal>1</ogc:Literal>
</ogc:PropertyIsLessThanOrEqualTo>
</ogc:Filter>
<PointSymbolizer>
<Graphic>
<Mark>
<WellKnownName>circle</WellKnownName>
<Fill>
<CssParameter name="fill">#FF0000</CssParameter>
</Fill>
</Mark>
<Size>8</Size>
</Graphic>
</PointSymbolizer>
</Rule>
<Rule>
<Name>rule29</Name>
<Title>2-9 Sites</Title>
<ogc:Filter>
<ogc:PropertyIsBetween>
<ogc:PropertyName>count</ogc:PropertyName>
<ogc:LowerBoundary>
<ogc:Literal>2</ogc:Literal>
</ogc:LowerBoundary>
<ogc:UpperBoundary>
<ogc:Literal>9</ogc:Literal>
</ogc:UpperBoundary>
</ogc:PropertyIsBetween>
</ogc:Filter>
<PointSymbolizer>
<Graphic>
<Mark>
<WellKnownName>circle</WellKnownName>
<Fill>
<CssParameter name="fill">#AA0000</CssParameter>
</Fill>
</Mark>
<Size>14</Size>
</Graphic>
</PointSymbolizer>
<TextSymbolizer>
<Label>
<ogc:PropertyName>count</ogc:PropertyName>
</Label>
<Font>
<CssParameter name="font-family">Arial</CssParameter>
<CssParameter name="font-size">12</CssParameter>
<CssParameter name="font-weight">bold</CssParameter>
</Font>
<LabelPlacement>
<PointPlacement>
<AnchorPoint>
<AnchorPointX>0.5</AnchorPointX>
<AnchorPointY>0.8</AnchorPointY>
</AnchorPoint>
</PointPlacement>
</LabelPlacement>
<Halo>
<Radius>2</Radius>
<Fill>
<CssParameter name="fill">#AA0000</CssParameter>
<CssParameter name="fill-opacity">0.9</CssParameter>
</Fill>
</Halo>
<Fill>
<CssParameter name="fill">#FFFFFF</CssParameter>
<CssParameter name="fill-opacity">1.0</CssParameter>
</Fill>
</TextSymbolizer>
</Rule>
<Rule>
<Name>rule10</Name>
<Title>10 Sites</Title>
<ogc:Filter>
<ogc:PropertyIsGreaterThan>
<ogc:PropertyName>count</ogc:PropertyName>
<ogc:Literal>9</ogc:Literal>
</ogc:PropertyIsGreaterThan>
</ogc:Filter>
<PointSymbolizer>
<Graphic>
<Mark>
<WellKnownName>circle</WellKnownName>
<Fill>
<CssParameter name="fill">#AA0000</CssParameter>
</Fill>
</Mark>
<Size>22</Size>
</Graphic>
</PointSymbolizer>
<TextSymbolizer>
<Label>
<ogc:PropertyName>count</ogc:PropertyName>
</Label>
<Font>
<CssParameter name="font-family">Arial</CssParameter>
<CssParameter name="font-size">12</CssParameter>
<CssParameter name="font-weight">bold</CssParameter>
</Font>
<LabelPlacement>
<PointPlacement>
<AnchorPoint>
<AnchorPointX>0.5</AnchorPointX>
<AnchorPointY>0.8</AnchorPointY>
</AnchorPoint>
</PointPlacement>
</LabelPlacement>
<Halo>
<Radius>2</Radius>
<Fill>
<CssParameter name="fill">#AA0000</CssParameter>
<CssParameter name="fill-opacity">0.9</CssParameter>
</Fill>
</Halo>
<Fill>
<CssParameter name="fill">#FFFFFF</CssParameter>
<CssParameter name="fill-opacity">1.0</CssParameter>
</Fill>
</TextSymbolizer>
</Rule>
</FeatureTypeStyle>
</UserStyle>
</NamedLayer>
</StyledLayerDescriptor>

View File

@ -17,6 +17,7 @@ import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest; import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import org.junit.Test; import org.junit.Test;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assume.assumeTrue; import static org.junit.Assume.assumeTrue;
public class IndexerTest extends BasicPluginTest { public class IndexerTest extends BasicPluginTest {
@ -37,11 +38,12 @@ public class IndexerTest extends BasicPluginTest {
IndexerPluginInterface plugin = (IndexerPluginInterface) plugins.get(SDIIndexerPlugin.DESCRIPTOR.getId()); IndexerPluginInterface plugin = (IndexerPluginInterface) plugins.get(SDIIndexerPlugin.DESCRIPTOR.getId());
IndexDocumentReport response = plugin.index(request); IndexDocumentReport response = plugin.index(request);
assumeTrue(response!=null); assertTrue(response!=null);
response.validate(); response.validate();
System.out.println("Response is "+Serialization.write(response));
assumeTrue(response.getStatus().equals(Report.Status.OK)); assertTrue(response.getStatus().equals(Report.Status.OK));
assumeTrue(response.prepareResult().getSpatialReference()!=null); assertTrue(response.prepareResult().getSpatialReference()!=null);
} }
} }