gCubeSystem
/
gcube-cms-suite
15
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactored tests

This commit is contained in:
Fabio Sinibaldi 2022-11-09 16:55:39 +01:00
parent 936c2f92c5
commit 5442e727df
9 changed files with 182 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
cms-test-commons/src/main/java/org/gcube/application/cms/tests/model/BasicTests.java
View File

@ -2,6 +2,7 @@ package org.gcube.application.cms.tests.model;
import com.vdurmont.semver4j.Semver;
import org.bson.Document;
import org.gcube.application.cms.tests.TokenSetter;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
@ -14,14 +15,24 @@ import org.gcube.application.geoportal.common.model.plugins.LifecycleManagerDesc
import org.gcube.application.geoportal.common.model.useCaseDescriptor.DataAccessPolicy;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import org.junit.BeforeClass;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.UUID;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assume.assumeTrue;
public class BasicTests {
public class BasicTests extends GCubeTest {
@BeforeClass
public static void initContext(){
assumeTrue(isTestInfrastructureEnabled());
TokenSetter.set(getContext());
}
protected User getCurrentUser(){
User u= new User();
@ -30,7 +41,7 @@ public class BasicTests {
}
protected UseCaseDescriptor getUCD(){
return initUCD(getContext(),getCurrentUser());
return initUCD(getContextObject(),getCurrentUser());
}
protected Document getBasicDocument(){
@ -38,10 +49,10 @@ public class BasicTests {
}
protected Project getBasicProject(){
return initProject(getBasicDocument(),getUCD(), getCurrentUser(), getContext());
return initProject(getBasicDocument(),getUCD(), getCurrentUser(), getContextObject());
}
protected Context getContext(){
protected Context getContextObject(){
Context toReturn = new Context();
toReturn.setName("My Fake Vre");
toReturn.setId("FAKE-VRE");
@ -49,7 +60,7 @@ public class BasicTests {
}
protected PublicationInfo getCurrentInfo(){
return initPublicationInfo(getUCD(),getContext(),getCurrentUser());
return initPublicationInfo(getUCD(),getContextObject(),getCurrentUser());
}
protected static PublicationInfo initPublicationInfo(UseCaseDescriptor ucd, Context ctx, User user){

4
cms-test-commons/src/main/java/org/gcube/application/cms/tests/plugins/BasicLCPluginTest.java
View File

@ -66,11 +66,11 @@ public class BasicLCPluginTest extends BasicPluginTest{
protected StepExecutionRequest prepareStepRequest(String stepId){
return new StepExecutionRequest(getUCD(),getCurrentUser(),getContext(),getBasicProject(),stepId);
return new StepExecutionRequest(getUCD(),getCurrentUser(),getContextObject(),getBasicProject(),stepId);
};
protected EventExecutionRequest prepareEventRequest(String event){
return new EventExecutionRequest(getUCD(),getCurrentUser(),getContext(),getBasicProject(),event);
return new EventExecutionRequest(getUCD(),getCurrentUser(),getContextObject(),getBasicProject(),event);
};

7
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/Constants.java Normal file
View File

@ -0,0 +1,7 @@
package org.gcube.application.cms.sdi;
public class Constants {
public final static String INDEXER_PLUGIN_ID="SDI-Indexer-Plugin";
}

28
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java
View File

@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.CrossReferencedLayer;
import org.gcube.application.cms.sdi.plugins.SDIIndexerPlugin;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
@ -33,6 +34,18 @@ public class PostgisIndexer {
};
// STD Fields
public static class StandardFields{
public static final PostgisTable.Field PROJECT_ID= new PostgisTable.Field(DBConstants.Defaults.PROJECT_ID, PostgisTable.FieldType.TEXT);
public static final PostgisTable.Field GEOM= new PostgisTable.Field(DBConstants.Defaults.DEFAULT_GEOMETRY_COLUMN_NAME, PostgisTable.FieldType.GEOMETRY);
public static final PostgisTable.Field DISPLAY=new PostgisTable.Field(DBConstants.Defaults.DISPLAYED,PostgisTable.FieldType.BOOLEAN);
}
@NonNull
SDIManagerWrapper manager;
@NonNull
@ -49,7 +62,7 @@ public class PostgisIndexer {
private List<CrossReferencedLayer> crossReferenceableLayers= new ArrayList<>();
public PostgisIndexer(SDIManagerWrapper manager, UseCaseDescriptor useCaseDescriptor,
public PostgisIndexer(SDIManagerWrapper manager, UseCaseDescriptor useCaseDescriptor,
DatabaseConnection postgisConnection) throws SQLException {
log.info("POSTGIS Index for {} Connecting to {} ", useCaseDescriptor.getId(),postgisConnection);
this.connectionParameters=postgisConnection;
@ -65,9 +78,18 @@ public class PostgisIndexer {
public void initIndex(String indexName, List<PostgisTable.Field> fields, String workspace,String storeName) throws SQLException, SDIInteractionException {
public void initIndex(String indexName, List<PostgisTable.Field> customFields, String workspace,String storeName) throws SQLException, SDIInteractionException {
log.info("Check/init index for {} ", useCaseDescriptor.getId());
table = new PostgisTable(indexName,fields, PostgisTable.GeometryType.POINT);
List<PostgisTable.Field> tableFields= new ArrayList<>();
tableFields.add(StandardFields.GEOM);
tableFields.add(StandardFields.PROJECT_ID);
tableFields.add(StandardFields.DISPLAY);
tableFields.addAll(customFields);
table = new PostgisTable(indexName,tableFields, PostgisTable.GeometryType.POINT);
log.trace("Index Postgis Table is {} ",table);
log.debug("Create if missing..");
// Check if table exists

18
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java
View File

@ -3,17 +3,16 @@ package org.gcube.application.cms.sdi.engine;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.cms.sdi.faults.DataParsingException;
import org.gcube.application.cms.sdi.model.MappingObject;
import org.gcube.application.geoportal.common.model.legacy.BBOX;
import java.sql.Array;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -50,6 +49,17 @@ public class PostgisTable {
private FieldType type;
private Boolean isIndexed;
private Object constantValue;
public static final Field fromMapping(MappingObject m){
return new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType()));
}
public static final List<Field> fromMappings (Collection<MappingObject> coll){
ArrayList<Field> toReturn = new ArrayList<>();
if(coll!=null)
coll.forEach(m -> toReturn.add(fromMapping(m)));
return toReturn;
}
}
@Getter

2
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java
View File

@ -279,7 +279,7 @@ public class SDIManagerWrapper extends SDIManager{
}
private static class CQL_GS_Feature extends GSFeatureTypeEncoder{
private static final String CQL = "CQL";
private static final String CQL = "cqlFilter";
public void setCQL(String cql){this.set(CQL,cql);}
}

45
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/MappingObject.java Normal file
View File

@ -0,0 +1,45 @@
package org.gcube.application.cms.sdi.model;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.plugins.faults.InvalidProfileException;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import java.util.ArrayList;
import java.util.List;
@Data
@Slf4j
public class MappingObject {
private String name;
private String type;
private String path;
public void validate() throws InvalidProfileException {
if (name == null) throw new InvalidProfileException("Invalid mapping " + this + " : name is null");
if (type == null) throw new InvalidProfileException("Invalid mapping " + this + " : type is null");
if (path == null) throw new InvalidProfileException("Invalid mapping " + this + " : path is null");
}
public static List<MappingObject> getMappingsFromUCD(UseCaseDescriptor ucd, String handlerID) throws InvalidProfileException {
log.debug("UseCaseDescriptor {} : Evaluating Index schema.. ", ucd.getId());
HandlerDeclaration handler = ucd.getHandlersMapByID().get(handlerID).get(0);
log.trace("Handler is {} ",handler);
Document profileConfiguration = handler.getConfiguration();
List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class);
log.trace("Loading mappings from useCaseDescriptor.. ");
List<MappingObject> mappingObjects= new ArrayList<>();
if(mappingObjs!=null){
for (Object mappingObj : mappingObjs) {
log.trace("Mapping is {} ",mappingObj);
MappingObject m = Serialization.convert(mappingObj,MappingObject.class);
m.validate();
mappingObjects.add(m);
}
}
return mappingObjects;
}
}

66
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java
View File

@ -1,7 +1,6 @@
package org.gcube.application.cms.sdi.plugins;
import com.vdurmont.semver4j.Semver;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.plugins.IndexerPluginInterface;
@ -14,6 +13,7 @@ import org.gcube.application.cms.plugins.reports.InitializationReport;
import org.gcube.application.cms.plugins.reports.Report;
import org.gcube.application.cms.plugins.requests.BaseRequest;
import org.gcube.application.cms.plugins.requests.IndexDocumentRequest;
import org.gcube.application.cms.sdi.Constants;
import org.gcube.application.cms.sdi.engine.DBConstants;
import org.gcube.application.cms.sdi.engine.PostgisIndexer;
import org.gcube.application.cms.sdi.engine.PostgisTable;
@ -21,6 +21,7 @@ import org.gcube.application.cms.sdi.engine.bboxes.BBOXByCoordinatePaths;
import org.gcube.application.cms.sdi.engine.bboxes.BBOXEvaluator;
import org.gcube.application.cms.sdi.engine.bboxes.BBOXPathScanner;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.MappingObject;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.configuration.Index;
@ -45,24 +46,13 @@ import java.util.List;
@Slf4j
public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPluginInterface {
@Data
private static class MappingObject{
private String name;
private String type;
private String path;
public void validate () throws InvalidProfileException {
if(name==null) throw new InvalidProfileException("Invalid mapping "+this+" : name is null");
if(type==null) throw new InvalidProfileException("Invalid mapping "+this+" : type is null");
if(path==null) throw new InvalidProfileException("Invalid mapping "+this+" : path is null");
}
}
static final PluginDescriptor DESCRIPTOR=new PluginDescriptor("SDI-Indexer-Plugin",
static final PluginDescriptor DESCRIPTOR=new PluginDescriptor(Constants.INDEXER_PLUGIN_ID,
IndexerPluginDescriptor.INDEXER);
static final ArrayList<BBOXEvaluator> BBOX_EVALUATORS=new ArrayList<>();
static {
@ -266,13 +256,17 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
}
}
private List<MappingObject> getMappings(UseCaseDescriptor useCaseDescriptor) throws InvalidProfileException {
return MappingObject.getMappingsFromUCD(useCaseDescriptor, getDescriptor().getId());
}
@Override
public IndexDocumentReport deindex(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Serving Index Request {} ",this.getDescriptor().getId(),request);
IndexDocumentReport report= new IndexDocumentReport(request);
try{
PostgisIndexer indexer = getIndexer(request.getUseCaseDescriptor(),request.getCallParameters());
indexer.removeByFieldValue(Fields.PROJECT_ID,request.getDocument().getId());
indexer.removeByFieldValue(PostgisIndexer.StandardFields.PROJECT_ID,request.getDocument().getId());
}catch (SDIInteractionException e){
log.error("Unable to index "+request,e);
report.setStatus(Report.Status.ERROR);
@ -310,8 +304,7 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
PostgisIndexer indexer = new PostgisIndexer(sdiCache.getObject(), ucd, postgisCache.getObject());
List<MappingObject> mappingObjects = getMappings(ucd);
List<PostgisTable.Field> fields = getFields(mappingObjects);
List<PostgisTable.Field> fields = PostgisTable.Field.fromMappings(mappingObjects);
indexer.initIndex(params.getString("indexName"),
fields,
@ -321,44 +314,11 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
}
private static class Fields{
public static final PostgisTable.Field PROJECT_ID= new PostgisTable.Field(DBConstants.Defaults.PROJECT_ID, PostgisTable.FieldType.TEXT);
public static final PostgisTable.Field GEOM= new PostgisTable.Field(DBConstants.Defaults.DEFAULT_GEOMETRY_COLUMN_NAME, PostgisTable.FieldType.GEOMETRY);
public static final PostgisTable.Field DISPLAY=new PostgisTable.Field(DBConstants.Defaults.DISPLAYED,PostgisTable.FieldType.BOOLEAN);
}
private List<PostgisTable.Field> getFields(List<MappingObject> mappings){
List<PostgisTable.Field> fields = new ArrayList<>(); // TODO From UseCaseDescriptor
fields.add(Fields.GEOM);
fields.add(Fields.PROJECT_ID);
fields.add(Fields.DISPLAY);
mappings.forEach(m -> {
fields.add(new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType())));
});
return fields;
}
private List<MappingObject> getMappings(UseCaseDescriptor ucd) throws InvalidProfileException {
log.debug("UseCaseDescriptor {} : Evaluating Index schema.. ", ucd.getId());
Document profileConfiguration = getConfigurationFromProfile(ucd).getConfiguration();
List mappingObjs= profileConfiguration.get("explicitFieldMapping",List.class);
log.trace("Loading mappings from useCaseDescriptor.. ");
List<MappingObject> mappingObjects= new ArrayList<>();
if(mappingObjs!=null){
for (Object mappingObj : mappingObjs) {
log.trace("Mapping is {} ",mappingObj);
MappingObject m = Serialization.convert(mappingObj,MappingObject.class);
m.validate();
mappingObjects.add(m);
}
}
return mappingObjects;
}
}

59
sdi-plugins/src/test/java/org/gcube/application/cms/sdi/LayerCreationTest.java Normal file
View File

@ -0,0 +1,59 @@
package org.gcube.application.cms.sdi;
import org.gcube.application.cms.implementations.ISInterface;
import org.gcube.application.cms.implementations.ImplementationProvider;
import org.gcube.application.cms.plugins.faults.InvalidProfileException;
import org.gcube.application.cms.sdi.engine.PostgisIndexer;
import org.gcube.application.cms.sdi.engine.PostgisTable;
import org.gcube.application.cms.sdi.engine.SDIManagerWrapper;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.MappingObject;
import org.gcube.application.cms.sdi.plugins.SDIIndexerPlugin;
import org.gcube.application.cms.tests.TestProfiles;
import org.gcube.application.cms.tests.model.BasicTests;
import org.gcube.application.cms.tests.plugins.BasicPluginTest;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import org.joda.time.Instant;
import org.joda.time.format.DateTimeFormatter;
import org.junit.Test;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.List;
import static org.junit.Assume.assumeTrue;
public class LayerCreationTest extends BasicTests {
@Test
public void testCreateIndexLayer() throws SDIInteractionException, ConfigurationException, SQLException, InvalidProfileException {
assumeTrue(GCubeTest.isTestInfrastructureEnabled());
UseCaseDescriptor ucd = TestProfiles.profiles.get("sdi-tests");
DatabaseConnection db = ImplementationProvider.get().getProvidedObjectByClass(ISInterface.class).
queryForDatabase("Database","postgis", "GNA_DB","Concessioni");
SDIManagerWrapper wrapper = new SDIManagerWrapper();
PostgisIndexer indexer = new PostgisIndexer(wrapper, ucd, db);
List<MappingObject> mappingObjects = MappingObject.getMappingsFromUCD(ucd, Constants.INDEXER_PLUGIN_ID);
List<PostgisTable.Field> fields = PostgisTable.Field.fromMappings(mappingObjects);
String indexName = "test_"+PostgisTable.sanitizeFieldName(Instant.now().toString());
String ws = "testWS";
indexer.initIndex(indexName,
fields,
ws,
indexName);
System.out.println("Done "+ws);
}
}