Added apply regex business logic [#26322]

This commit is contained in:
Francesco Mangiacrapa 2023-12-21 16:38:49 +01:00
parent 07d74c68e4
commit b1e4f70afb
8 changed files with 220 additions and 31 deletions

View File

@ -1,5 +1,8 @@
# Changelog for org.gcube.application.cms.sdi-plugins # Changelog for org.gcube.application.cms.sdi-plugins
## [v1.1.3-SNAPSHOT]
- Added apply regex business logic [#26322]
## [v1.1.2] ## [v1.1.2]
- Using parent version range [#25572] - Using parent version range [#25572]

View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<artifactId>sdi-plugins</artifactId> <artifactId>sdi-plugins</artifactId>
<version>1.1.2</version> <version>1.1.3-SNAPSHOT</version>
<name>gCube CMS - SDI Plugins</name> <name>gCube CMS - SDI Plugins</name>
<parent> <parent>

View File

@ -0,0 +1,40 @@
package org.gcube.application.cms.sdi.model;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
/**
* The Class ApplyRegex.
*
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
*
* Dec 21, 2023
*
* e.g.
* {
* "apply_regex":
* { "type": "replaceAll",
* "regex": "(\\s)?\\([\\s\\S]*",
* "replacement": ""
* }
* }
*/
@Data
@Slf4j
public class ApplyRegex {
/**
* The Enum REGEX_TYPES.
*
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
*
* Dec 21, 2023
*/
public static enum REGEX_TYPES {replaceAll, replaceFirst, find}
String type;
String regex;
String replacement = "";
}

View File

@ -9,6 +9,8 @@ import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration; import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor; import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -18,7 +20,11 @@ public class MappingObject {
private String name; private String name;
private String type; private String type;
private String path; private String path;
//Added by Francesco M.
//Here we can pass a regex to manage the value that must be added to index
@JsonProperty("apply_regex")
private ApplyRegex applyRegex;
public void validate() throws InvalidProfileException { public void validate() throws InvalidProfileException {
if (name == null) throw new InvalidProfileException("Invalid mapping " + this + " : name is null"); if (name == null) throw new InvalidProfileException("Invalid mapping " + this + " : name is null");
if (type == null) throw new InvalidProfileException("Invalid mapping " + this + " : type is null"); if (type == null) throw new InvalidProfileException("Invalid mapping " + this + " : type is null");

View File

@ -5,6 +5,8 @@ import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.bson.Document; import org.bson.Document;
import org.gcube.application.cms.plugins.IndexerPluginInterface; import org.gcube.application.cms.plugins.IndexerPluginInterface;
@ -26,6 +28,8 @@ import org.gcube.application.cms.sdi.engine.bboxes.BBOXByCoordinatePaths;
import org.gcube.application.cms.sdi.engine.bboxes.BBOXEvaluator; import org.gcube.application.cms.sdi.engine.bboxes.BBOXEvaluator;
import org.gcube.application.cms.sdi.engine.bboxes.BBOXPathScanner; import org.gcube.application.cms.sdi.engine.bboxes.BBOXPathScanner;
import org.gcube.application.cms.sdi.faults.SDIInteractionException; import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.ApplyRegex;
import org.gcube.application.cms.sdi.model.ApplyRegex.REGEX_TYPES;
import org.gcube.application.cms.sdi.model.MappingObject; import org.gcube.application.cms.sdi.model.MappingObject;
import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.JSONPathWrapper; import org.gcube.application.geoportal.common.model.JSONPathWrapper;
@ -50,14 +54,14 @@ import com.vdurmont.semver4j.Semver;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
/** /**
* The Class SDIIndexerPlugin. * The Class SDIIndexerPlugin.
* *
* @author created by Fabio Sinibaldi * @author created by Fabio Sinibaldi
* @author new architect and maintainer - Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it * @author new architect and maintainer - Francesco Mangiacrapa at ISTI-CNR
* francesco.mangiacrapa@isti.cnr.it
* *
* Apr 28, 2023 * Apr 28, 2023
*/ */
@Slf4j @Slf4j
public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPluginInterface { public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPluginInterface {
@ -141,7 +145,7 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
centroidDoc.put(DBConstants.Defaults.PROJECT_ID, project.getId()); centroidDoc.put(DBConstants.Defaults.PROJECT_ID, project.getId());
centroidDoc.put(DBConstants.Defaults.DISPLAYED, true); centroidDoc.put(DBConstants.Defaults.DISPLAYED, true);
//*********** Added by Francesco. Creating Gis Viewer Link as public or private // *********** Added by Francesco. Creating Gis Viewer Link as public or private
Boolean isInternalIndex = null; Boolean isInternalIndex = null;
try { try {
isInternalIndex = requestArguments.getBoolean(IndexConstants.INDEX_PARAMETER_FLAGINTERNALINDEX); isInternalIndex = requestArguments.getBoolean(IndexConstants.INDEX_PARAMETER_FLAGINTERNALINDEX);
@ -173,41 +177,44 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
log.error("Error on creating the Geoportal GisViewer link for project id {}", project.getId(), e); log.error("Error on creating the Geoportal GisViewer link for project id {}", project.getId(), e);
} }
} }
// ********************** Updated by Francesco, see #25056, Calculating Spatial Reference as Centroid Object // ********************** Updated by Francesco, see #25056, Calculating Spatial
//**** EVALUATE POSITION // Reference as Centroid Object
// **** EVALUATE POSITION
log.debug("indexing UseCaseDescriptor {} : Evaluating Centroid... ", useCaseDescriptor.getId()); log.debug("indexing UseCaseDescriptor {} : Evaluating Centroid... ", useCaseDescriptor.getId());
SpatialReference reference = null; SpatialReference reference = null;
List<IdentificationReference> refs = project List<IdentificationReference> refs = project
.getIdentificationReferenceByType(SpatialReference.SPATIAL_REFERENCE_TYPE); .getIdentificationReferenceByType(SpatialReference.SPATIAL_REFERENCE_TYPE);
GCubeSDILayer.BBOX bbox = null; GCubeSDILayer.BBOX bbox = null;
if (!refs.isEmpty()) { if (!refs.isEmpty()) {
// Use existing Reference // Use existing Reference
try { try {
reference = Serialization.convert(refs.get(0), SpatialReference.class); reference = Serialization.convert(refs.get(0), SpatialReference.class);
log.debug("Using already defined spatial reference " + reference); log.debug("Using already defined spatial reference " + reference);
GeoJsonObject object = Serialization.convert(reference.getGeoJson(), GeoJsonObject.class); GeoJsonObject object = Serialization.convert(reference.getGeoJson(), GeoJsonObject.class);
bbox = GCubeSDILayer.BBOX.fromGeoJSON(object.getBbox()); bbox = GCubeSDILayer.BBOX.fromGeoJSON(object.getBbox());
log.info("Found declared BBOX {} ", bbox); log.info("Found declared BBOX {} ", bbox);
Double pointX = (bbox.getMaxX() + bbox.getMinX()) / 2; Double pointX = (bbox.getMaxX() + bbox.getMinX()) / 2;
Double pointY = (bbox.getMaxY() + bbox.getMinY()) / 2; Double pointY = (bbox.getMaxY() + bbox.getMinY()) / 2;
String wkt = String.format("POINT (%1$f %2$f) ", pointX, pointY); String wkt = String.format("POINT (%1$f %2$f) ", pointX, pointY);
centroidDoc.put("geom", wkt); centroidDoc.put("geom", wkt);
//No overriding SpatialReference in the IndexDocumentReport, using the existing one. // No overriding SpatialReference in the IndexDocumentReport, using the existing
}catch (Exception e) { // one.
} catch (Exception e) {
log.info("The defined spatial reference is wrong or empty: " + reference); log.info("The defined spatial reference is wrong or empty: " + reference);
} }
} }
//Calculating and overriding the SpatialReference (the Centroid JSON) in the IndexDocumentReport. // Calculating and overriding the SpatialReference (the Centroid JSON) in the
if(bbox==null) { // IndexDocumentReport.
if (bbox == null) {
log.info("No bbox defined in the current spatial reference going to calculate it..."); log.info("No bbox defined in the current spatial reference going to calculate it...");
// unable to use current Spatial reference, try evaluating it // unable to use current Spatial reference, try evaluating it
log.debug("UseCaseDescriptor {} : Getting evaluation paths from useCaseDescriptor.. ", log.debug("UseCaseDescriptor {} : Getting evaluation paths from useCaseDescriptor.. ",
@ -275,6 +282,9 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
b.deleteCharAt(b.length() - 1); b.deleteCharAt(b.length() - 1);
toSetValue = b.toString(); toSetValue = b.toString();
} }
// Added by Francesco M. #26322
toSetValue = toSetValueByApplyRegex(m, toSetValue);
log.trace("Setting {} = {} in centroid doc ", m.getName(), toSetValue); log.trace("Setting {} = {} in centroid doc ", m.getName(), toSetValue);
centroidDoc.put(m.getName(), toSetValue); centroidDoc.put(m.getName(), toSetValue);
} }
@ -311,6 +321,43 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
} }
} }
// Added by Francesco M. #26322
public static Object toSetValueByApplyRegex(MappingObject m, Object toSetValue) {
ApplyRegex applyRegex = m.getApplyRegex();
if (applyRegex != null) {
try {
String valueString = toSetValue==null?"":toSetValue.toString();
String type = applyRegex.getType();
REGEX_TYPES theRegexType = ApplyRegex.REGEX_TYPES.valueOf(type);
Pattern p = Pattern.compile(applyRegex.getRegex());
Matcher matcher = p.matcher(valueString);
switch (theRegexType) {
case find:
while (matcher.find()) {
toSetValue += matcher.group();
}
break;
case replaceFirst:
toSetValue = matcher.replaceFirst(applyRegex.getReplacement());
break;
case replaceAll:
toSetValue = matcher.replaceFirst(applyRegex.getReplacement());
break;
default:
break;
}
} catch (Exception e) {
log.warn("Error on applying replaceAll by regex {} on field {} = {} in centroid doc ", applyRegex,
m.getName(), toSetValue, e);
}
}
return toSetValue;
}
/** /**
* Gets the mappings. * Gets the mappings.
* *
@ -338,13 +385,14 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
try { try {
PostgisIndexer indexer = getIndexer(request.getUseCaseDescriptor(), request.getCallParameters()); PostgisIndexer indexer = getIndexer(request.getUseCaseDescriptor(), request.getCallParameters());
indexer.removeByFieldValue(PostgisIndexer.StandardFields.PROJECT_ID, request.getDocument().getId()); indexer.removeByFieldValue(PostgisIndexer.StandardFields.PROJECT_ID, request.getDocument().getId());
//***** Added by Francesco, see #25056 // ***** Added by Francesco, see #25056
//Replacing the "old" centroid if any. Creating an empty SpatialReference // Replacing the "old" centroid if any. Creating an empty SpatialReference
SpatialReference reference = new SpatialReference(new Document()); SpatialReference reference = new SpatialReference(new Document());
log.info("DeIndexer project {}, Setting Spatial Reference empty {} ", request.getDocument().getId(), Serialization.write(reference)); log.info("DeIndexer project {}, Setting Spatial Reference empty {} ", request.getDocument().getId(),
Serialization.write(reference));
report.addIdentificationReference(reference); report.addIdentificationReference(reference);
} catch (SDIInteractionException e) { } catch (SDIInteractionException e) {
log.error("Unable to index " + request, e); log.error("Unable to index " + request, e);
report.setStatus(Report.Status.ERROR); report.setStatus(Report.Status.ERROR);
@ -379,11 +427,11 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
/** /**
* Gets the indexer. * Gets the indexer.
* *
* @param ucd the ucd * @param ucd the ucd
* @param params the params * @param params the params
* @return the indexer * @return the indexer
* @throws ConfigurationException the configuration exception * @throws ConfigurationException the configuration exception
* @throws SQLException the SQL exception * @throws SQLException the SQL exception
* @throws InvalidProfileException the invalid profile exception * @throws InvalidProfileException the invalid profile exception
* @throws SDIInteractionException the SDI interaction exception * @throws SDIInteractionException the SDI interaction exception
*/ */

View File

@ -23,12 +23,11 @@ import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import org.junit.Test; import org.junit.Test;
public class LayerCreationTest extends BasicTests { public class LayerCreationTest extends BasicTests {
@Test @Test
public void testCreateIndexLayer() throws SDIInteractionException, ConfigurationException, SQLException, InvalidProfileException { public void testCreateIndexLayer() throws SDIInteractionException, ConfigurationException, SQLException, InvalidProfileException {
assumeTrue(GCubeTest.isTestInfrastructureEnabled()); assumeTrue(GCubeTest.isTestInfrastructureEnabled());
UseCaseDescriptor ucd = TestProfiles.profiles.get("sdi-tests"); UseCaseDescriptor ucd = TestProfiles.profiles.get("concessioni");
DatabaseConnection db = ImplementationProvider.get().getProvidedObjectByClass(ISInterface.class). DatabaseConnection db = ImplementationProvider.get().getProvidedObjectByClass(ISInterface.class).
queryForDatabase("Database","postgis", "GNA_DB","Concessioni"); queryForDatabase("Database","postgis", "GNA_DB","Concessioni");

View File

@ -0,0 +1,50 @@
package org.gcube.application.cms.sdi;
import static org.junit.Assume.assumeTrue;
import java.sql.SQLException;
import java.util.List;
import java.util.stream.Collectors;
import org.gcube.application.cms.plugins.faults.InvalidProfileException;
import org.gcube.application.cms.sdi.engine.PostgisTable;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.MappingObject;
import org.gcube.application.cms.sdi.plugins.SDIIndexerPlugin;
import org.gcube.application.cms.tests.TestProfiles;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import org.junit.Test;
public class MappingObjectRegexTest {
String test = "paleolitico inferiore (da 2.000.000 a.C. al 200.000 a.C.)";
@Test
public void testApplyRegex() throws SDIInteractionException, ConfigurationException, SQLException,
InvalidProfileException, InterruptedException {
assumeTrue(GCubeTest.isTestInfrastructureEnabled());
UseCaseDescriptor ucd = TestProfiles.profiles.get("profiledConcessioni");
System.out.println("ucd: " + ucd);
List<MappingObject> mappingObjects = MappingObject.getMappingsFromUCD(ucd, Constants.INDEXER_PLUGIN_ID);
String toPrint = mappingObjects.stream().map(mo -> String.valueOf(mo)).collect(Collectors.joining("\n"));
System.out.println(toPrint);
for (MappingObject mappingObject : mappingObjects) {
Object toSetValue = SDIIndexerPlugin.toSetValueByApplyRegex(mappingObject, test);
System.out.println("MappingObject '" + mappingObject.getName() + "' toSetValue is: " + toSetValue);
}
List<PostgisTable.Field> fields = PostgisTable.Field.fromMappings(mappingObjects);
for (PostgisTable.Field field : fields) {
System.out.println(field);
}
}
}

View File

@ -0,0 +1,43 @@
package org.gcube.application.cms.sdi;
import static org.junit.Assume.assumeTrue;
import java.sql.SQLException;
import java.util.List;
import java.util.stream.Collectors;
import org.gcube.application.cms.plugins.faults.InvalidProfileException;
import org.gcube.application.cms.sdi.engine.PostgisTable;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.MappingObject;
import org.gcube.application.cms.tests.TestProfiles;
import org.gcube.application.cms.tests.model.BasicTests;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import org.junit.Test;
public class MappingsFromUCDTest extends BasicTests {
@Test
public void testReadMappingsFromUCD()
throws SDIInteractionException, ConfigurationException, SQLException, InvalidProfileException {
assumeTrue(GCubeTest.isTestInfrastructureEnabled());
UseCaseDescriptor ucd = TestProfiles.profiles.get("profiledConcessioni");
System.out.println("ucd: " + ucd);
List<MappingObject> mappingObjects = MappingObject.getMappingsFromUCD(ucd, Constants.INDEXER_PLUGIN_ID);
String toPrint = mappingObjects.stream().map(mo -> String.valueOf(mo)).collect(Collectors.joining("\n"));
System.out.println(toPrint);
List<PostgisTable.Field> fields = PostgisTable.Field.fromMappings(mappingObjects);
for (PostgisTable.Field field : fields) {
System.out.println(field);
}
}
}