diff --git a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/model/BasicTests.java b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/model/BasicTests.java index 06ca9fe..8736fa5 100644 --- a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/model/BasicTests.java +++ b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/model/BasicTests.java @@ -2,6 +2,7 @@ package org.gcube.application.cms.tests.model; import com.vdurmont.semver4j.Semver; import org.bson.Document; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.geoportal.common.model.document.Project; import org.gcube.application.geoportal.common.model.document.access.Access; import org.gcube.application.geoportal.common.model.document.access.AccessPolicy; @@ -14,14 +15,24 @@ import org.gcube.application.geoportal.common.model.plugins.LifecycleManagerDesc import org.gcube.application.geoportal.common.model.useCaseDescriptor.DataAccessPolicy; import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration; import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor; +import org.gcube.application.geoportal.common.utils.tests.GCubeTest; +import org.junit.BeforeClass; import java.time.LocalDateTime; import java.util.Collections; import java.util.UUID; import static junit.framework.TestCase.assertTrue; +import static org.junit.Assume.assumeTrue; -public class BasicTests { +public class BasicTests extends GCubeTest { + + + @BeforeClass + public static void initContext(){ + assumeTrue(isTestInfrastructureEnabled()); + TokenSetter.set(getContext()); + } protected User getCurrentUser(){ User u= new User(); @@ -30,7 +41,7 @@ public class BasicTests { } protected UseCaseDescriptor getUCD(){ - return initUCD(getContext(),getCurrentUser()); + return initUCD(getContextObject(),getCurrentUser()); } protected Document getBasicDocument(){ @@ -38,10 +49,10 @@ public class BasicTests { } protected Project getBasicProject(){ - return initProject(getBasicDocument(),getUCD(), getCurrentUser(), getContext()); + return initProject(getBasicDocument(),getUCD(), getCurrentUser(), getContextObject()); } - protected Context getContext(){ + protected Context getContextObject(){ Context toReturn = new Context(); toReturn.setName("My Fake Vre"); toReturn.setId("FAKE-VRE"); @@ -49,7 +60,7 @@ public class BasicTests { } protected PublicationInfo getCurrentInfo(){ - return initPublicationInfo(getUCD(),getContext(),getCurrentUser()); + return initPublicationInfo(getUCD(),getContextObject(),getCurrentUser()); } protected static PublicationInfo initPublicationInfo(UseCaseDescriptor ucd, Context ctx, User user){ diff --git a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicLCPluginTest.java b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicLCPluginTest.java index 17c334d..3250e42 100644 --- a/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicLCPluginTest.java +++ b/cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicLCPluginTest.java @@ -66,11 +66,11 @@ public class BasicLCPluginTest extends BasicPluginTest{ protected StepExecutionRequest prepareStepRequest(String stepId){ - return new StepExecutionRequest(getUCD(),getCurrentUser(),getContext(),getBasicProject(),stepId); + return new StepExecutionRequest(getUCD(),getCurrentUser(),getContextObject(),getBasicProject(),stepId); }; protected EventExecutionRequest prepareEventRequest(String event){ - return new EventExecutionRequest(getUCD(),getCurrentUser(),getContext(),getBasicProject(),event); + return new EventExecutionRequest(getUCD(),getCurrentUser(),getContextObject(),getBasicProject(),event); }; diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/Constants.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/Constants.java new file mode 100644 index 0000000..77aab2f --- /dev/null +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/Constants.java @@ -0,0 +1,7 @@ +package org.gcube.application.cms.sdi; + +public class Constants { + + public final static String INDEXER_PLUGIN_ID="SDI-Indexer-Plugin"; + +} diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java index 135a057..035a199 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java @@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.gcube.application.cms.sdi.faults.SDIInteractionException; import org.gcube.application.cms.sdi.model.CrossReferencedLayer; +import org.gcube.application.cms.sdi.plugins.SDIIndexerPlugin; import org.gcube.application.geoportal.common.model.configuration.Index; import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; @@ -33,6 +34,18 @@ public class PostgisIndexer { }; + + // STD Fields + public static class StandardFields{ + public static final PostgisTable.Field PROJECT_ID= new PostgisTable.Field(DBConstants.Defaults.PROJECT_ID, PostgisTable.FieldType.TEXT); + public static final PostgisTable.Field GEOM= new PostgisTable.Field(DBConstants.Defaults.DEFAULT_GEOMETRY_COLUMN_NAME, PostgisTable.FieldType.GEOMETRY); + + public static final PostgisTable.Field DISPLAY=new PostgisTable.Field(DBConstants.Defaults.DISPLAYED,PostgisTable.FieldType.BOOLEAN); + + } + + + @NonNull SDIManagerWrapper manager; @NonNull @@ -49,7 +62,7 @@ public class PostgisIndexer { private List crossReferenceableLayers= new ArrayList<>(); - public PostgisIndexer(SDIManagerWrapper manager, UseCaseDescriptor useCaseDescriptor, + public PostgisIndexer(SDIManagerWrapper manager, UseCaseDescriptor useCaseDescriptor, DatabaseConnection postgisConnection) throws SQLException { log.info("POSTGIS Index for {} Connecting to {} ", useCaseDescriptor.getId(),postgisConnection); this.connectionParameters=postgisConnection; @@ -65,9 +78,18 @@ public class PostgisIndexer { - public void initIndex(String indexName, List fields, String workspace,String storeName) throws SQLException, SDIInteractionException { + + public void initIndex(String indexName, List customFields, String workspace,String storeName) throws SQLException, SDIInteractionException { log.info("Check/init index for {} ", useCaseDescriptor.getId()); - table = new PostgisTable(indexName,fields, PostgisTable.GeometryType.POINT); + List tableFields= new ArrayList<>(); + tableFields.add(StandardFields.GEOM); + tableFields.add(StandardFields.PROJECT_ID); + tableFields.add(StandardFields.DISPLAY); + + tableFields.addAll(customFields); + + + table = new PostgisTable(indexName,tableFields, PostgisTable.GeometryType.POINT); log.trace("Index Postgis Table is {} ",table); log.debug("Create if missing.."); // Check if table exists diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java index 68cdf2d..fd20a07 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java @@ -3,17 +3,16 @@ package org.gcube.application.cms.sdi.engine; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.gcube.application.cms.sdi.faults.DataParsingException; +import org.gcube.application.cms.sdi.model.MappingObject; import org.gcube.application.geoportal.common.model.legacy.BBOX; +import java.sql.Array; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; import java.text.DecimalFormat; import java.text.NumberFormat; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -50,6 +49,17 @@ public class PostgisTable { private FieldType type; private Boolean isIndexed; private Object constantValue; + + public static final Field fromMapping(MappingObject m){ + return new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType())); + } + + public static final List fromMappings (Collection coll){ + ArrayList toReturn = new ArrayList<>(); + if(coll!=null) + coll.forEach(m -> toReturn.add(fromMapping(m))); + return toReturn; + } } @Getter diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java index bd6d31f..6a483d3 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java @@ -279,7 +279,7 @@ public class SDIManagerWrapper extends SDIManager{ } private static class CQL_GS_Feature extends GSFeatureTypeEncoder{ - private static final String CQL = "CQL"; + private static final String CQL = "cqlFilter"; public void setCQL(String cql){this.set(CQL,cql);} } diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/MappingObject.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/MappingObject.java new file mode 100644 index 0000000..01b2e28 --- /dev/null +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/MappingObject.java @@ -0,0 +1,45 @@ +package org.gcube.application.cms.sdi.model; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.gcube.application.cms.plugins.faults.InvalidProfileException; +import org.gcube.application.cms.serialization.Serialization; +import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration; +import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Slf4j +public class MappingObject { + private String name; + private String type; + private String path; + + public void validate() throws InvalidProfileException { + if (name == null) throw new InvalidProfileException("Invalid mapping " + this + " : name is null"); + if (type == null) throw new InvalidProfileException("Invalid mapping " + this + " : type is null"); + if (path == null) throw new InvalidProfileException("Invalid mapping " + this + " : path is null"); + } + + public static List getMappingsFromUCD(UseCaseDescriptor ucd, String handlerID) throws InvalidProfileException { + log.debug("UseCaseDescriptor {} : Evaluating Index schema.. ", ucd.getId()); + HandlerDeclaration handler = ucd.getHandlersMapByID().get(handlerID).get(0); + log.trace("Handler is {} ",handler); + Document profileConfiguration = handler.getConfiguration(); + List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class); + log.trace("Loading mappings from useCaseDescriptor.. "); + List mappingObjects= new ArrayList<>(); + if(mappingObjs!=null){ + for (Object mappingObj : mappingObjs) { + log.trace("Mapping is {} ",mappingObj); + MappingObject m = Serialization.convert(mappingObj,MappingObject.class); + m.validate(); + mappingObjects.add(m); + } + } + return mappingObjects; + } +} diff --git a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java index d3d3197..fa80841 100644 --- a/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java +++ b/sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java @@ -1,7 +1,6 @@ package org.gcube.application.cms.sdi.plugins; import com.vdurmont.semver4j.Semver; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.gcube.application.cms.plugins.IndexerPluginInterface; @@ -14,6 +13,7 @@ import org.gcube.application.cms.plugins.reports.InitializationReport; import org.gcube.application.cms.plugins.reports.Report; import org.gcube.application.cms.plugins.requests.BaseRequest; import org.gcube.application.cms.plugins.requests.IndexDocumentRequest; +import org.gcube.application.cms.sdi.Constants; import org.gcube.application.cms.sdi.engine.DBConstants; import org.gcube.application.cms.sdi.engine.PostgisIndexer; import org.gcube.application.cms.sdi.engine.PostgisTable; @@ -21,6 +21,7 @@ import org.gcube.application.cms.sdi.engine.bboxes.BBOXByCoordinatePaths; import org.gcube.application.cms.sdi.engine.bboxes.BBOXEvaluator; import org.gcube.application.cms.sdi.engine.bboxes.BBOXPathScanner; import org.gcube.application.cms.sdi.faults.SDIInteractionException; +import org.gcube.application.cms.sdi.model.MappingObject; import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.geoportal.common.model.JSONPathWrapper; import org.gcube.application.geoportal.common.model.configuration.Index; @@ -45,24 +46,13 @@ import java.util.List; @Slf4j public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPluginInterface { - @Data - private static class MappingObject{ - private String name; - private String type; - private String path; - public void validate () throws InvalidProfileException { - if(name==null) throw new InvalidProfileException("Invalid mapping "+this+" : name is null"); - if(type==null) throw new InvalidProfileException("Invalid mapping "+this+" : type is null"); - if(path==null) throw new InvalidProfileException("Invalid mapping "+this+" : path is null"); - } - } - - - - static final PluginDescriptor DESCRIPTOR=new PluginDescriptor("SDI-Indexer-Plugin", + static final PluginDescriptor DESCRIPTOR=new PluginDescriptor(Constants.INDEXER_PLUGIN_ID, IndexerPluginDescriptor.INDEXER); + + + static final ArrayList BBOX_EVALUATORS=new ArrayList<>(); static { @@ -266,13 +256,17 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin } } + private List getMappings(UseCaseDescriptor useCaseDescriptor) throws InvalidProfileException { + return MappingObject.getMappingsFromUCD(useCaseDescriptor, getDescriptor().getId()); + } + @Override public IndexDocumentReport deindex(IndexDocumentRequest request) throws InvalidPluginRequestException { log.info("Indexer {} : Serving Index Request {} ",this.getDescriptor().getId(),request); IndexDocumentReport report= new IndexDocumentReport(request); try{ PostgisIndexer indexer = getIndexer(request.getUseCaseDescriptor(),request.getCallParameters()); - indexer.removeByFieldValue(Fields.PROJECT_ID,request.getDocument().getId()); + indexer.removeByFieldValue(PostgisIndexer.StandardFields.PROJECT_ID,request.getDocument().getId()); }catch (SDIInteractionException e){ log.error("Unable to index "+request,e); report.setStatus(Report.Status.ERROR); @@ -310,8 +304,7 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin PostgisIndexer indexer = new PostgisIndexer(sdiCache.getObject(), ucd, postgisCache.getObject()); List mappingObjects = getMappings(ucd); - List fields = getFields(mappingObjects); - + List fields = PostgisTable.Field.fromMappings(mappingObjects); indexer.initIndex(params.getString("indexName"), fields, @@ -321,44 +314,11 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin } - private static class Fields{ - public static final PostgisTable.Field PROJECT_ID= new PostgisTable.Field(DBConstants.Defaults.PROJECT_ID, PostgisTable.FieldType.TEXT); - public static final PostgisTable.Field GEOM= new PostgisTable.Field(DBConstants.Defaults.DEFAULT_GEOMETRY_COLUMN_NAME, PostgisTable.FieldType.GEOMETRY); - - public static final PostgisTable.Field DISPLAY=new PostgisTable.Field(DBConstants.Defaults.DISPLAYED,PostgisTable.FieldType.BOOLEAN); - - } - private List getFields(List mappings){ - List fields = new ArrayList<>(); // TODO From UseCaseDescriptor - fields.add(Fields.GEOM); - fields.add(Fields.PROJECT_ID); - fields.add(Fields.DISPLAY); - mappings.forEach(m -> { - fields.add(new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType()))); - }); - - return fields; - } - private List getMappings(UseCaseDescriptor ucd) throws InvalidProfileException { - log.debug("UseCaseDescriptor {} : Evaluating Index schema.. ", ucd.getId()); - Document profileConfiguration = getConfigurationFromProfile(ucd).getConfiguration(); - List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class); - log.trace("Loading mappings from useCaseDescriptor.. "); - List mappingObjects= new ArrayList<>(); - if(mappingObjs!=null){ - for (Object mappingObj : mappingObjs) { - log.trace("Mapping is {} ",mappingObj); - MappingObject m = Serialization.convert(mappingObj,MappingObject.class); - m.validate(); - mappingObjects.add(m); - } - } - return mappingObjects; - } + } diff --git a/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/LayerCreationTest.java b/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/LayerCreationTest.java new file mode 100644 index 0000000..ed76f92 --- /dev/null +++ b/sdi-plugins/src/test/java/org/gcube/application/cms/sdi/LayerCreationTest.java @@ -0,0 +1,59 @@ +package org.gcube.application.cms.sdi; + +import org.gcube.application.cms.implementations.ISInterface; +import org.gcube.application.cms.implementations.ImplementationProvider; +import org.gcube.application.cms.plugins.faults.InvalidProfileException; +import org.gcube.application.cms.sdi.engine.PostgisIndexer; +import org.gcube.application.cms.sdi.engine.PostgisTable; +import org.gcube.application.cms.sdi.engine.SDIManagerWrapper; +import org.gcube.application.cms.sdi.faults.SDIInteractionException; +import org.gcube.application.cms.sdi.model.MappingObject; +import org.gcube.application.cms.sdi.plugins.SDIIndexerPlugin; +import org.gcube.application.cms.tests.TestProfiles; +import org.gcube.application.cms.tests.model.BasicTests; +import org.gcube.application.cms.tests.plugins.BasicPluginTest; +import org.gcube.application.geoportal.common.model.rest.ConfigurationException; +import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; +import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor; +import org.gcube.application.geoportal.common.utils.tests.GCubeTest; +import org.joda.time.Instant; +import org.joda.time.format.DateTimeFormatter; +import org.junit.Test; + +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.List; + +import static org.junit.Assume.assumeTrue; + +public class LayerCreationTest extends BasicTests { + + + @Test + public void testCreateIndexLayer() throws SDIInteractionException, ConfigurationException, SQLException, InvalidProfileException { + assumeTrue(GCubeTest.isTestInfrastructureEnabled()); + UseCaseDescriptor ucd = TestProfiles.profiles.get("sdi-tests"); + DatabaseConnection db = ImplementationProvider.get().getProvidedObjectByClass(ISInterface.class). + queryForDatabase("Database","postgis", "GNA_DB","Concessioni"); + + SDIManagerWrapper wrapper = new SDIManagerWrapper(); + + PostgisIndexer indexer = new PostgisIndexer(wrapper, ucd, db); + + List mappingObjects = MappingObject.getMappingsFromUCD(ucd, Constants.INDEXER_PLUGIN_ID); + List fields = PostgisTable.Field.fromMappings(mappingObjects); + + String indexName = "test_"+PostgisTable.sanitizeFieldName(Instant.now().toString()); + String ws = "testWS"; + + + indexer.initIndex(indexName, + fields, + ws, + indexName); + System.out.println("Done "+ws); + } + + + +}