gCubeSystem
/
gcube-cms-suite
13
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

task_24859 #13

Merged
francesco.mangiacrapa merged 14 commits from task_24859 into master 2023-03-30 11:00:17 +02:00
Conversation 0 Commits 14 Files Changed 26 +777 -677
26 changed files with 777 additions and 677 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
default-lc-managers/CHANGELOG.md
View File

@ -1,5 +1,10 @@
# Changelog for org.gcube.application.cms.ckan-plugin
## [v1.2.0-SNAPSHOT]
- Integrated the field 'geov_link' (Geoportal GisViewer link) in the centroid layer [#24859]
- Fixed Draft execution step executing dematerialization and de-indexing actions [#24877]
## [v1.1.1] - 2023-03-06
- [#24570] Integrated the UnPublish operation

2
default-lc-managers/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>default-lc-managers</artifactId>
<version>1.1.1</version>
<version>1.2.0-SNAPSHOT</version>
<name>gCube CMS - Default LC Managers</name>

2
default-lc-managers/src/main/java/org/gcube/application/cms/plugins/implementations/AbstractLifeCycleManager.java
View File

@ -41,7 +41,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractLifeCycleManager extends AbstractPlugin implements LifecycleManager {
protected static class Events{
public static final OperationDescriptor INIT=new OperationDescriptor(EventExecutionRequest.Events.ON_INIT_DOCUMENT,"Sets defaults and validate");
public static final OperationDescriptor UPDATE=new OperationDescriptor(EventExecutionRequest.Events.ON_UPDATE_DOCUMENT,"Sets defaults and validate");

89
default-lc-managers/src/main/java/org/gcube/application/cms/plugins/implementations/Default3PhaseManager.java
View File

@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Default3PhaseManager extends SimpleLifeCycleManager implements LifecycleManager {
protected static class Phases {
protected static class Phases {
public static final String PENDING_APPROVAL="Pending Approval";
public static final String PUBLISHED=SimpleLifeCycleManager.Phases.PUBLISHED;
@ -171,42 +171,69 @@ public class Default3PhaseManager extends SimpleLifeCycleManager implements Life
protected Document getInternalIndexParams(BaseRequest req){
Document callParameters = new Document();
callParameters.put("workspace", Files.fixFilename(req.getUseCaseDescriptor().getId()+"_internal_"+req.getContext().getName()));
callParameters.put("indexName",Files.fixFilename(req.getUseCaseDescriptor().getId()+"_internal_"+req.getContext().getName()+"_centroids"));
callParameters.put(IndexConstants.INDEX_PARAMETER_WORKSPACE, Files.fixFilename(req.getUseCaseDescriptor().getId()+"_internal_"+req.getContext().getName()));
callParameters.put(IndexConstants.INDEX_PARAMETER_INDEXNAME,Files.fixFilename(req.getUseCaseDescriptor().getId()+"_internal_"+req.getContext().getName()+"_centroids"));
//Added by Francesco
callParameters.put(IndexConstants.INDEX_PARAMETER_FLAGINTERNALINDEX, Boolean.TRUE);
return callParameters;
}
protected StepExecutionReport executeSubmit(StepExecutionReport theReport) throws Exception {
// Materialize
protected StepExecutionReport executeSubmit(StepExecutionReport theReport) throws Exception {
log.info(STEPS.SUBMIT + " running...");
for(MaterializationPlugin mat : getMaterializers(theReport.getTheRequest()))
theReport = materializeDocument(theReport,mat,getMaterializationParameters(theReport.getTheRequest()));
if(theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)){
// Index
for(IndexerPluginInterface indexer : getIndexers(theReport.getTheRequest()))
theReport = index(theReport,indexer,getInternalIndexParams(theReport.getTheRequest()));
// setPhase
if(theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
theReport.getToSetLifecycleInformation().setPhase(Phases.PENDING_APPROVAL);
}
return theReport;
}
// Materialize
for (MaterializationPlugin mat : getMaterializers(theReport.getTheRequest()))
theReport = materializeDocument(theReport, mat, getMaterializationParameters(theReport.getTheRequest()));
if (theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
// Index
for (IndexerPluginInterface indexer : getIndexers(theReport.getTheRequest()))
theReport = index(theReport, indexer, getInternalIndexParams(theReport.getTheRequest()));
// setPhase
if (theReport.getToSetLifecycleInformation().getLastOperationStatus()
.equals(LifecycleInformation.Status.OK))
theReport.getToSetLifecycleInformation().setPhase(Phases.PENDING_APPROVAL);
}
return theReport;
}
protected StepExecutionReport executeApprove(StepExecutionReport theReport) throws Exception {
// Index
for(IndexerPluginInterface indexer : getIndexers(theReport.getTheRequest()))
theReport = index(theReport,indexer,getPublicIndexParams(theReport.getTheRequest()));
// setPhase
if(theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
theReport.getToSetLifecycleInformation().setPhase(Phases.PUBLISHED);
return theReport;
}
protected StepExecutionReport executeApprove(StepExecutionReport theReport) throws Exception {
log.info(STEPS.APPROVE + " running...");
protected StepExecutionReport executeReject(StepExecutionReport theReport) throws Exception {
if(theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
theReport.getToSetLifecycleInformation().setPhase(LifecycleInformation.CommonPhases.DRAFT_PHASE);
return theReport;
}
// Index
for (IndexerPluginInterface indexer : getIndexers(theReport.getTheRequest()))
theReport = index(theReport, indexer, getPublicIndexParams(theReport.getTheRequest()));
// setPhase
if (theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
theReport.getToSetLifecycleInformation().setPhase(Phases.PUBLISHED);
return theReport;
}
//Updated by Francesco, see #24877
protected StepExecutionReport executeReject(StepExecutionReport theReport) throws Exception {
log.info(STEPS.REJECT + " running...");
// DeMaterialize
JSONPathWrapper wrapper = new JSONPathWrapper(theReport.getTheRequest().getDocument().getTheDocument().toJson());
for (String s : wrapper.getMatchingPaths("$..[?(@." + RegisteredFileSet.PAYLOADS + ")]")) {
log.info("Requesting dematerialization for {} ", s);
for (MaterializationPlugin mat : getMaterializers(theReport.getTheRequest()))
theReport = deMaterialize(theReport, mat, new Document("fileSetPath", s));
if (!theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
break;
}
// DeIndexing
if (theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) {
for (IndexerPluginInterface indexer : getIndexers(theReport.getTheRequest())) {
log.info("Requesting deindexing for {} ", indexer.getDescriptor());
theReport = deIndex(theReport, indexer, getInternalIndexParams(theReport.getTheRequest()));
}
}
if (theReport.getToSetLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
theReport.getToSetLifecycleInformation().setPhase(LifecycleInformation.CommonPhases.DRAFT_PHASE);
return theReport;
}
}

16
default-lc-managers/src/main/java/org/gcube/application/cms/plugins/implementations/IndexConstants.java Normal file
View File

@ -0,0 +1,16 @@
package org.gcube.application.cms.plugins.implementations;
/**
* The Class IndexConstants.
*
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
*
* Mar 29, 2023
*/
public class IndexConstants {
public static final String INDEX_PARAMETER_FLAGINTERNALINDEX = "flagInternalIndex";
public static final String INDEX_PARAMETER_INDEXNAME = "indexName";
public static final String INDEX_PARAMETER_WORKSPACE = "workspace";
}

5
default-lc-managers/src/main/java/org/gcube/application/cms/plugins/implementations/SimpleLifeCycleManager.java
View File

@ -93,8 +93,9 @@ public class SimpleLifeCycleManager extends AbstractLifeCycleManager implements
protected Document getPublicIndexParams(BaseRequest req){
Document callParameters = new Document();
callParameters.put("workspace",Files.fixFilename(req.getUseCaseDescriptor().getId()+req.getContext().getName()));
callParameters.put("indexName",Files.fixFilename(req.getUseCaseDescriptor().getId()+req.getContext().getName()+"_centroids"));
callParameters.put(IndexConstants.INDEX_PARAMETER_WORKSPACE,Files.fixFilename(req.getUseCaseDescriptor().getId()+req.getContext().getName()));
callParameters.put(IndexConstants.INDEX_PARAMETER_INDEXNAME,Files.fixFilename(req.getUseCaseDescriptor().getId()+req.getContext().getName()+"_centroids"));
callParameters.put(IndexConstants.INDEX_PARAMETER_FLAGINTERNALINDEX, Boolean.FALSE);
return callParameters;
}

3
geoportal-service/CHANGELOG.md
View File

@ -1,5 +1,8 @@
# Changelog for org.gcube.application.geoportal-service
## [v1.0.14-SNAPSHOT]
- Fixed deploying of multiple versions of the default-lc-managers [#24875]
## [v1.0.13] - 2023-02-23
- Integrating new facilities and bug fixing released in the Plugins

9
geoportal-service/pom.xml
View File

@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.gcube.application</groupId>
<artifactId>geoportal-service</artifactId>
<version>1.0.13</version>
<version>1.0.14-SNAPSHOT</version>
<name>Geoportal Service</name>
<packaging>war</packaging>
@ -21,16 +21,12 @@
<webappDirectory>${project.basedir}${file.separator}${project.artifactId}${file.separator}src${file.separator}main${file.separator}webapp${file.separator}WEB-INF</webappDirectory>
</properties>
<scm>
<connection>scm:git:${gitBaseUrl}/gcube-cms-suite</connection>
<developerConnection>scm:git:${gitBaseUrl}/gcube-cms-suite</developerConnection>
<url>${gitBaseUrl}/gcube-cms-suite</url>
</scm>
<dependencyManagement>
<dependencies>
<dependency>
@ -94,7 +90,6 @@
</dependency>
<!-- enunciate deps -->
<dependency>
<groupId>com.webcohesion.enunciate</groupId>
@ -173,7 +168,7 @@
<build>
<plugins>
<!-- SPHINX PLUGIN triggered at 'compile'-->
<!-- SPHINX PLUGIN triggered at 'compile' -->
<plugin>
<groupId>kr.motd.maven</groupId>
<artifactId>sphinx-maven-plugin</artifactId>

4
sdi-plugins/CHANGELOG.md
View File

@ -1,5 +1,9 @@
# Changelog for org.gcube.application.cms.sdi-plugins
## [v1.1.0-SNAPSHOT]
- Integrated the field 'geov_link' (Geoportal GisViewer link) in the centroid layer [#24859]
## [v1.0.4] - 2023-03-06
- Fixed the import from joda-time to java.time
- [#24702] Fixed the default-lc-managers dependency

9
sdi-plugins/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>sdi-plugins</artifactId>
<version>1.0.4</version>
<version>1.1.0-SNAPSHOT</version>
<name>gCube CMS - SDI Plugins</name>
@ -85,12 +85,19 @@
<version>1.14</version>
</dependency>
<dependency>
<groupId>org.gcube.portlets.user</groupId>
<artifactId>uri-resolver-manager</artifactId>
<version>[1.0.0,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.application.cms</groupId>
<artifactId>cms-test-commons</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

6
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/DBConstants.java
View File

@ -2,7 +2,9 @@ package org.gcube.application.cms.sdi.engine;
import java.util.ArrayList;
import static org.gcube.application.cms.sdi.engine.PostgisTable.*;
import org.gcube.application.cms.sdi.engine.PostgisTable.Field;
import org.gcube.application.cms.sdi.engine.PostgisTable.FieldType;
import org.gcube.application.cms.sdi.engine.PostgisTable.GeometryType;
public class DBConstants {
@ -18,6 +20,8 @@ public class DBConstants {
public static final String PROJECT_ID="projectid";
public static final String DISPLAYED="displayed_project";
public static final String GEOVIEWER_LINK_FIELD="geov_link";
public static final String XCOORD_FIELD="xcoord";

9
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisDBManager.java
View File

@ -1,9 +1,5 @@
package org.gcube.application.cms.sdi.engine;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.cms.sdi.faults.DataParsingException;
import org.gcube.application.geoportal.common.model.legacy.BBOX;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@ -11,6 +7,11 @@ import java.sql.SQLException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.gcube.application.cms.sdi.faults.DataParsingException;
import org.gcube.application.geoportal.common.model.legacy.BBOX;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class PostgisDBManager implements PostgisDBManagerI {

26
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java
View File

@ -1,16 +1,5 @@
package org.gcube.application.cms.sdi.engine;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.CrossReferencedLayer;
import org.gcube.application.cms.sdi.plugins.SDIIndexerPlugin;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@ -18,6 +7,17 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.bson.Document;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.CrossReferencedLayer;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class PostgisIndexer {
@ -39,8 +39,8 @@ public class PostgisIndexer {
public static class StandardFields{
public static final PostgisTable.Field PROJECT_ID= new PostgisTable.Field(DBConstants.Defaults.PROJECT_ID, PostgisTable.FieldType.TEXT);
public static final PostgisTable.Field GEOM= new PostgisTable.Field(DBConstants.Defaults.DEFAULT_GEOMETRY_COLUMN_NAME, PostgisTable.FieldType.GEOMETRY);
public static final PostgisTable.Field DISPLAY=new PostgisTable.Field(DBConstants.Defaults.DISPLAYED,PostgisTable.FieldType.BOOLEAN);
public static final PostgisTable.Field GEOV_LINK=new PostgisTable.Field(DBConstants.Defaults.GEOVIEWER_LINK_FIELD,PostgisTable.FieldType.TEXT);
}
@ -85,10 +85,10 @@ public class PostgisIndexer {
tableFields.add(StandardFields.GEOM);
tableFields.add(StandardFields.PROJECT_ID);
tableFields.add(StandardFields.DISPLAY);
tableFields.add(StandardFields.GEOV_LINK);
tableFields.addAll(customFields);
table = new PostgisTable(indexName,tableFields, PostgisTable.GeometryType.POINT);
log.trace("Index Postgis Table is {} ",table);
log.debug("Create if missing..");

548
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisTable.java
View File

@ -1,20 +1,29 @@
package org.gcube.application.cms.sdi.engine;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.gcube.application.cms.sdi.faults.DataParsingException;
import org.gcube.application.cms.sdi.model.MappingObject;
import org.gcube.application.geoportal.common.model.legacy.BBOX;
import java.sql.Array;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@ -22,315 +31,314 @@ import java.util.regex.Pattern;
@ToString
public class PostgisTable {
@Getter
@AllArgsConstructor
public static enum GeometryType {
MULTIPOINT("4326", "geometry (MULTIPOINT,4326)", "", ""), POINT("4326", "geometry (POINT,4326)", "", ""),
LINE("4326", "geometry (MULTILINESTRING,4326)", "", ""),
POLYGON("4326", "geometry (MULTIPOLYGON,4326)", "", "");
@Getter
@AllArgsConstructor
public static enum GeometryType{
MULTIPOINT("4326","geometry (MULTIPOINT,4326)","",""),
POINT("4326","geometry (POINT,4326)","",""),
LINE("4326","geometry (MULTILINESTRING,4326)","",""),
POLYGON("4326","geometry (MULTIPOLYGON,4326)","","");
private final String SRID;
private final String definition;
private final String InsertWKT;
private final String insertWKB;
private final String SRID;
private final String definition;
private final String InsertWKT;
private final String insertWKB;
}
}
@RequiredArgsConstructor
@Getter
@Setter
@ToString
public static class Field {
@NonNull
private String name;
@NonNull
private FieldType type;
private Boolean isIndexed;
private Object constantValue;
@RequiredArgsConstructor
@Getter
@Setter
@ToString
public static class Field{
@NonNull
private String name;
@NonNull
private FieldType type;
private Boolean isIndexed;
private Object constantValue;
public static final Field fromMapping(MappingObject m) {
return new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType()));
}
public static final Field fromMapping(MappingObject m){
return new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType()));
}
public static final List<Field> fromMappings(Collection<MappingObject> coll) {
ArrayList<Field> toReturn = new ArrayList<>();
if (coll != null)
coll.forEach(m -> toReturn.add(fromMapping(m)));
return toReturn;
}
}
public static final List<Field> fromMappings (Collection<MappingObject> coll){
ArrayList<Field> toReturn = new ArrayList<>();
if(coll!=null)
coll.forEach(m -> toReturn.add(fromMapping(m)));
return toReturn;
}
}
@Getter
@AllArgsConstructor
public enum FieldType {
INT("int", java.sql.Types.INTEGER), BOOLEAN("boolean", java.sql.Types.BOOLEAN),
TEXT("text", java.sql.Types.LONGVARCHAR), FLOAT("float", java.sql.Types.FLOAT), GEOMETRY("", 0),
AUTOINCREMENT("BIGSERIAL PRIMARY KEY", java.sql.Types.BIGINT);
@Getter
@AllArgsConstructor
public enum FieldType{
INT("int",java.sql.Types.INTEGER),
BOOLEAN("boolean", java.sql.Types.BOOLEAN),
TEXT("text",java.sql.Types.LONGVARCHAR),
FLOAT("float",java.sql.Types.FLOAT),
GEOMETRY("",0),
AUTOINCREMENT("BIGSERIAL PRIMARY KEY",java.sql.Types.BIGINT);
private String definition;
private int sqlType;
}
private String definition;
private int sqlType;
}
@RequiredArgsConstructor
@Getter
@ToString
public static class POINT {
private static Pattern pattern = Pattern.compile("(?!=\\d\\.\\d\\.)([\\d.]+)");
@RequiredArgsConstructor
@Getter
@ToString
public static class POINT{
public static POINT parsePOINT(String point) throws DataParsingException {
// POINT(8.30230113965909 44.8011688237011)
// x,y
try {
log.debug("Parsing POINT " + point);
Matcher m = pattern.matcher(point);
private static Pattern pattern = Pattern.compile("(?!=\\d\\.\\d\\.)([\\d.]+)");
if (!m.find())
throw new DataParsingException("Unable to get x ");
Double x = Double.parseDouble(m.group(1));
public static POINT parsePOINT(String point) throws DataParsingException {
//POINT(8.30230113965909 44.8011688237011)
// x,y
try {
log.debug("Parsing POINT "+point);
Matcher m=pattern.matcher(point);
if (!m.find())
throw new DataParsingException("Unable to get y ");
Double y = Double.parseDouble(m.group(1));
if(!m.find()) throw new DataParsingException("Unable to get x ");
Double x=Double.parseDouble(m.group(1));
return new POINT(x, y);
} catch (Throwable t) {
throw new DataParsingException("Invalid POINT " + point, t);
}
}
if(!m.find()) throw new DataParsingException("Unable to get y ");
Double y=Double.parseDouble(m.group(1));
@NonNull
private Double x;
@NonNull
private Double y;
return new POINT(x,y);
}catch(Throwable t) {
throw new DataParsingException("Invalid POINT "+point,t);
}
}
}
@NonNull
private Double x;
@NonNull
private Double y;
private static final NumberFormat DECIMAL_FORMAT = NumberFormat.getInstance(Locale.US);
}
static {
((DecimalFormat) DECIMAL_FORMAT).setGroupingUsed(false);
}
public String getGeometryColumn() {
for (Field f : fields)
if (f.getType().equals(FieldType.GEOMETRY))
return f.getName();
return null;
}
private static final NumberFormat DECIMAL_FORMAT=NumberFormat.getInstance(Locale.US);
@NonNull
private String tablename;
static {
((DecimalFormat) DECIMAL_FORMAT).setGroupingUsed(false);
}
@NonNull
private List<Field> fields;
@NonNull
private GeometryType geometryColumnType;
public String getGeometryColumn() {
for(Field f:fields)
if(f.getType().equals(FieldType.GEOMETRY)) return f.getName();
return null;
}
@Setter
private BBOX boundingBox = null;
@NonNull
private String tablename;
@Setter
private POINT centroid = null;
@NonNull
private List<Field> fields;
public void setTablename(String tablename) {
this.tablename = sanitizeFieldName(tablename);
}
public String getCreateStatement() {
StringBuilder stmt = new StringBuilder();
stmt.append("CREATE TABLE IF NOT EXISTS " + tablename + "( ");
for (Field field : fields) {
@NonNull
private GeometryType geometryColumnType;
String fieldDefinition = field.getType().getDefinition();
if (field.getType().equals(FieldType.GEOMETRY))
fieldDefinition = this.getGeometryColumnType().definition;
@Setter
private BBOX boundingBox=null;
stmt.append(field.getName() + " " + fieldDefinition + ",");
}
stmt.deleteCharAt(stmt.lastIndexOf(","));
stmt.append(")");
return stmt.toString();
}
@Setter
private POINT centroid=null;
public String getDeleteByFieldStatement(Field field) {
return "DELETE FROM " + tablename + " WHERE " + field.getName() + " = ? ";
}
public void setTablename(String tablename) {
this.tablename = sanitizeFieldName(tablename);
}
public String getInsertionStatement(boolean geometryText) {
StringBuilder fieldList = new StringBuilder();
StringBuilder fieldInsertion = new StringBuilder();
public String getCreateStatement() {
StringBuilder stmt=new StringBuilder();
stmt.append("CREATE TABLE IF NOT EXISTS "+tablename+"( ");
for(Field field:fields){
for (Field field : fields) {
switch (field.getType()) {
case AUTOINCREMENT:
break;
case GEOMETRY: {
fieldList.append(field.getName() + ",");
if (geometryText)
fieldInsertion.append("ST_GeomFromText(?, 4326),");
else
fieldInsertion.append("ST_GeomFromWKB(?, 4326),");
break;
}
default: {
fieldList.append(field.getName() + ",");
fieldInsertion.append("?,");
}
}
}
String fieldDefinition=field.getType().getDefinition();
if(field.getType().equals(FieldType.GEOMETRY))
fieldDefinition=this.getGeometryColumnType().definition;
fieldList.deleteCharAt(fieldList.lastIndexOf(","));
fieldInsertion.deleteCharAt(fieldInsertion.lastIndexOf(","));
stmt.append(field.getName()+" "+fieldDefinition+",");
}
stmt.deleteCharAt(stmt.lastIndexOf(","));
stmt.append(")");
return stmt.toString();
}
return "Insert into " + tablename + " (" + fieldList + ") VALUES (" + fieldInsertion + ")";
}
public String getDeleteByFieldStatement(Field field) {
return "DELETE FROM "+tablename+" WHERE "+field.getName()+" = ? ";
}
public void fillObjectsPreparedStatement(Map<String, Object> row, PreparedStatement toFill) throws SQLException {
int psFieldIndex = 0;
HashMap<String, Object> rowValues = new HashMap<String, Object>();
for (Map.Entry<String, Object> entry : row.entrySet())
rowValues.put(sanitizeFieldName(entry.getKey()), entry.getValue());
for (Field field : fields) {
if (!field.getType().equals(FieldType.AUTOINCREMENT)) {
psFieldIndex++;
public String getInsertionStatement(boolean geometryText) {
StringBuilder fieldList=new StringBuilder();
StringBuilder fieldInsertion=new StringBuilder();
Object value = rowValues.get(field.getName());
setObjectInPreparedStatement(field, value, toFill, psFieldIndex);
for(Field field:fields) {
switch(field.getType()) {
case AUTOINCREMENT : break;
case GEOMETRY : {
fieldList.append(field.getName()+",");
if(geometryText)
fieldInsertion.append("ST_GeomFromText(?, 4326),");
else
fieldInsertion.append("ST_GeomFromWKB(?, 4326),");
break;
}
default : {
fieldList.append(field.getName()+",");
fieldInsertion.append("?,");
}
}
}
}
fieldList.deleteCharAt(fieldList.lastIndexOf(","));
fieldInsertion.deleteCharAt(fieldInsertion.lastIndexOf(","));
}
}
public void setObjectInPreparedStatement(Field field, Object value, PreparedStatement toFill, int psFieldIndex)
throws SQLException {
if (value == null) {
try {
toFill.setNull(psFieldIndex, field.getType().sqlType);
} catch (SQLException e) {
log.error("Unable to set null for field " + field);
throw e;
}
} else {
switch (field.getType()) {
case FLOAT: {
toFill.setFloat(psFieldIndex, (Float) value);
break;
}
case INT: {
toFill.setInt(psFieldIndex, (Integer) value);
break;
}
case TEXT: {
toFill.setString(psFieldIndex, value.toString());
break;
}
case GEOMETRY: {
if (value instanceof String)
toFill.setString(psFieldIndex, ((String) value));
else
toFill.setBytes(psFieldIndex, (byte[]) value);
break;
}
case BOOLEAN: {
if (value instanceof String)
toFill.setBoolean(psFieldIndex, Boolean.parseBoolean(value.toString()));
if (value instanceof Boolean)
toFill.setBoolean(psFieldIndex, (Boolean) value);
break;
}
// Added by Francesco
default: {
if (value instanceof String) {
toFill.setString(psFieldIndex, ((String) value));
} else {
try {
String toStringValue = value.toString();
toFill.setString(psFieldIndex, toStringValue);
} catch (Exception e) {
// silence
}
}
}
}
}
}
return "Insert into "+tablename+" ("+fieldList+") VALUES ("+fieldInsertion+")";
}
public void fillCSVPreparedStatament(Map<String, String> row, PreparedStatement toFill, boolean explicitGeometry)
throws SQLException {
int psFieldIndex = 0;
HashMap<String, String> rowValues = new HashMap<String, String>();
for (Map.Entry<String, String> entry : row.entrySet())
rowValues.put(sanitizeFieldName(entry.getKey()), entry.getValue());
public void fillObjectsPreparedStatement(Map<String,Object> row, PreparedStatement toFill) throws SQLException {
int psFieldIndex=0;
HashMap<String,Object> rowValues=new HashMap<String,Object>();
for(Map.Entry<String,Object> entry:row.entrySet())
rowValues.put(sanitizeFieldName(entry.getKey()), entry.getValue());
for (Field field : fields) {
if (!field.getType().equals(FieldType.AUTOINCREMENT)) {
psFieldIndex++;
for(Field field:fields) {
if(!field.getType().equals(FieldType.AUTOINCREMENT)) {
psFieldIndex++;
Object value=rowValues.get(field.getName());
setObjectInPreparedStatement(field,value,toFill,psFieldIndex);
}
}
}
public void setObjectInPreparedStatement(Field field,Object value, PreparedStatement toFill, int psFieldIndex) throws SQLException {
if(value==null) {
try{
toFill.setNull(psFieldIndex, field.getType().sqlType);
}catch(SQLException e) {
log.error("Unable to set null for field "+field);
throw e;
}
}
else
switch(field.getType()) {
case FLOAT :{
toFill.setFloat(psFieldIndex, (Float)value);
break;
}
case INT : {
toFill.setInt(psFieldIndex, (Integer)value);
break;
}
case TEXT : {
toFill.setString(psFieldIndex, value.toString());
break;
}
case GEOMETRY : {
if(value instanceof String)
toFill.setString(psFieldIndex, ((String) value));
else toFill.setBytes(psFieldIndex, (byte[])value);
break;
}
case BOOLEAN: {
if(value instanceof String)
toFill.setBoolean(psFieldIndex,Boolean.parseBoolean(value.toString()));
if(value instanceof Boolean)
toFill.setBoolean(psFieldIndex,(Boolean) value);
break;
}
}
}
public void fillCSVPreparedStatament(Map<String,String> row, PreparedStatement toFill,boolean explicitGeometry) throws SQLException {
int psFieldIndex=0;
HashMap<String,String> rowValues=new HashMap<String,String>();
for(Map.Entry<String,String> entry:row.entrySet())
rowValues.put(sanitizeFieldName(entry.getKey()), entry.getValue());
for(Field field:fields) {
if(!field.getType().equals(FieldType.AUTOINCREMENT)) {
psFieldIndex++;
String value=rowValues.get(field.getName());
String value = rowValues.get(field.getName());
// if(value==null||value.equalsIgnoreCase("null")) toFill.setNull(psFieldIndex, field.getType().sqlType);
// else
switch(field.getType()) {
case FLOAT :{
try{
toFill.setFloat(psFieldIndex, Float.parseFloat(value));
}catch(NumberFormatException e) {
throw new SQLException(field+" cannot be null. CSV Row is "+rowValues,e);
}
break;
}
case INT : {
try{
toFill.setInt(psFieldIndex, Integer.parseInt(value));
}catch(NumberFormatException e) {
log.warn("Skipping value for "+field+" row was "+rowValues,e);
toFill.setNull(psFieldIndex, java.sql.Types.INTEGER);
}
break;
}
case TEXT : {
toFill.setString(psFieldIndex, value.toString());
break;
}
case GEOMETRY : {
if(explicitGeometry) {
toFill.setString(psFieldIndex,value);
}else {
switch(geometryColumnType){
case POINT: {
String xRepresentation=DECIMAL_FORMAT.format(Double.parseDouble(rowValues.get(DBConstants.Defaults.XCOORD_FIELD)));
String yRepresentation=DECIMAL_FORMAT.format(Double.parseDouble(rowValues.get(DBConstants.Defaults.YCOORD_FIELD)));
switch (field.getType()) {
case FLOAT: {
try {
toFill.setFloat(psFieldIndex, Float.parseFloat(value));
} catch (NumberFormatException e) {
throw new SQLException(field + " cannot be null. CSV Row is " + rowValues, e);
}
break;
}
case INT: {
try {
toFill.setInt(psFieldIndex, Integer.parseInt(value));
} catch (NumberFormatException e) {
log.warn("Skipping value for " + field + " row was " + rowValues, e);
toFill.setNull(psFieldIndex, java.sql.Types.INTEGER);
}
break;
}
case TEXT: {
toFill.setString(psFieldIndex, value.toString());
break;
}
case GEOMETRY: {
if (explicitGeometry) {
toFill.setString(psFieldIndex, value);
} else {
switch (geometryColumnType) {
case POINT: {
String xRepresentation = DECIMAL_FORMAT
.format(Double.parseDouble(rowValues.get(DBConstants.Defaults.XCOORD_FIELD)));
String yRepresentation = DECIMAL_FORMAT
.format(Double.parseDouble(rowValues.get(DBConstants.Defaults.YCOORD_FIELD)));
toFill.setString(psFieldIndex, "POINT("+xRepresentation+" "+
yRepresentation+")");
break;
}
default :{
toFill.setString(psFieldIndex,rowValues.get("wkt"));
break;
}
}
}
break;
}
}
}
toFill.setString(psFieldIndex, "POINT(" + xRepresentation + " " + yRepresentation + ")");
break;
}
default: {
toFill.setString(psFieldIndex, rowValues.get("wkt"));
break;
}
}
}
break;
}
}
}
}
}
}
}
public static String sanitizeFieldName(String fieldName) {
public static String sanitizeFieldName(String fieldName) {
// return fieldName.toLowerCase().replaceAll(" ", "_").replaceAll("\\.", "").replaceAll("-", "_").replaceAll("////","_");
return fieldName.toLowerCase().replaceAll("[^a-z0-9_\\\\]", "_");
}
return fieldName.toLowerCase().replaceAll("[^a-z0-9_\\\\]", "_");
}
}

19
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManager.java
View File

@ -1,21 +1,22 @@
package org.gcube.application.cms.sdi.engine;
import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.spatial.data.gis.GISInterface;
import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.spatial.data.gis.GISInterface;
import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor;
import it.geosolutions.geoserver.rest.encoder.datastore.GSPostGISDatastoreEncoder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SDIManager {

42
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java
View File

@ -1,13 +1,15 @@
package org.gcube.application.cms.sdi.engine;
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher;
import it.geosolutions.geoserver.rest.GeoServerRESTReader;
import it.geosolutions.geoserver.rest.decoder.RESTFeatureType;
import it.geosolutions.geoserver.rest.decoder.RESTLayer;
import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.GSAttributeEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder;
import lombok.extern.slf4j.Slf4j;
import static it.geosolutions.geoserver.rest.encoder.GSResourceEncoder.ProjectionPolicy.REPROJECT_TO_DECLARED;
import static org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder.EPSG_4326;
import static org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder.WGS84_FULL;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.bson.Document;
import org.gcube.application.cms.plugins.requests.BaseExecutionRequest;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
@ -21,23 +23,25 @@ import org.gcube.application.geoportal.common.model.document.filesets.sdi.GeoSer
import org.gcube.application.geoportal.common.model.document.filesets.sdi.PlatformInfo;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.client.Client;
import org.gcube.data.transfer.library.faults.*;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import static it.geosolutions.geoserver.rest.encoder.GSResourceEncoder.ProjectionPolicy.REPROJECT_TO_DECLARED;
import static org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder.EPSG_4326;
import static org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder.WGS84_FULL;
import it.geosolutions.geoserver.rest.GeoServerRESTPublisher;
import it.geosolutions.geoserver.rest.GeoServerRESTReader;
import it.geosolutions.geoserver.rest.decoder.RESTFeatureType;
import it.geosolutions.geoserver.rest.decoder.RESTLayer;
import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SDIManagerWrapper extends SDIManager{

9
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/bboxes/BBOXByCoordinatePaths.java
View File

@ -1,15 +1,16 @@
package org.gcube.application.cms.sdi.engine.bboxes;
import lombok.Data;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import org.bson.Document;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class BBOXByCoordinatePaths extends BBOXEvaluator{

9
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/bboxes/BBOXEvaluator.java
View File

@ -1,13 +1,14 @@
package org.gcube.application.cms.sdi.engine.bboxes;
import org.bson.Document;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
@RequiredArgsConstructor
@ToString

5
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/bboxes/BBOXPathScanner.java
View File

@ -1,13 +1,14 @@
package org.gcube.application.cms.sdi.engine.bboxes;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import org.bson.Document;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class BBOXPathScanner extends BBOXEvaluator{

13
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/GCubeSDILayerBuilder.java
View File

@ -1,16 +1,17 @@
package org.gcube.application.cms.sdi.model;
import lombok.Getter;
import org.bson.Document;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GeoServerPlatform;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.PlatformInfo;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.bson.Document;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GeoServerPlatform;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.PlatformInfo;
import lombok.Getter;
public class GCubeSDILayerBuilder {
public static enum OGC_TYPE{

9
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/MappingObject.java
View File

@ -1,15 +1,16 @@
package org.gcube.application.cms.sdi.model;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.List;
import org.bson.Document;
import org.gcube.application.cms.plugins.faults.InvalidProfileException;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.HandlerDeclaration;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Data
@Slf4j

4
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/SDIConfiguration.java
View File

@ -1,10 +1,10 @@
package org.gcube.application.cms.sdi.model;
import org.bson.Document;
import java.util.HashMap;
import java.util.Map;
import org.bson.Document;
public class SDIConfiguration {
public static final String GEOSERVER_PLATFORM="GeoServer";

7
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/SupportedFormat.java
View File

@ -1,11 +1,12 @@
package org.gcube.application.cms.sdi.model;
import java.util.ArrayList;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
import java.util.ArrayList;
@Getter
@RequiredArgsConstructor
public abstract class SupportedFormat {

9
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIAbstractPlugin.java
View File

@ -1,7 +1,8 @@
package org.gcube.application.cms.sdi.plugins;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import org.gcube.application.cms.caches.AbstractScopedMap;
import org.gcube.application.cms.implementations.ISInterface;
import org.gcube.application.cms.implementations.ImplementationProvider;
@ -16,8 +17,8 @@ import org.gcube.application.cms.sdi.engine.SDIManagerWrapper;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class SDIAbstractPlugin extends AbstractPlugin implements InitializablePlugin {

576
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java
View File

@ -1,13 +1,18 @@
package org.gcube.application.cms.sdi.plugins;
import com.vdurmont.semver4j.Semver;
import lombok.extern.slf4j.Slf4j;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.bson.Document;
import org.gcube.application.cms.plugins.IndexerPluginInterface;
import org.gcube.application.cms.plugins.faults.IndexingException;
import org.gcube.application.cms.plugins.faults.InitializationException;
import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException;
import org.gcube.application.cms.plugins.faults.InvalidProfileException;
import org.gcube.application.cms.plugins.implementations.IndexConstants;
import org.gcube.application.cms.plugins.reports.IndexDocumentReport;
import org.gcube.application.cms.plugins.reports.InitializationReport;
import org.gcube.application.cms.plugins.reports.Report;
@ -33,292 +38,303 @@ import org.gcube.application.geoportal.common.model.plugins.IndexerPluginDescrip
import org.gcube.application.geoportal.common.model.plugins.PluginDescriptor;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.portlets.user.uriresolvermanager.UriResolverManager;
import org.gcube.portlets.user.uriresolvermanager.resolvers.query.GeoportalResolverQueryStringBuilder;
import org.gcube.portlets.user.uriresolvermanager.resolvers.query.GeoportalResolverQueryStringBuilder.RESOLVE_AS;
import org.geojson.Crs;
import org.geojson.GeoJsonObject;
import org.geojson.LngLatAlt;
import org.geojson.Point;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import com.vdurmont.semver4j.Semver;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPluginInterface {
static final PluginDescriptor DESCRIPTOR=new PluginDescriptor(Constants.INDEXER_PLUGIN_ID,
IndexerPluginDescriptor.INDEXER);
static final ArrayList<BBOXEvaluator> BBOX_EVALUATORS=new ArrayList<>();
static {
DESCRIPTOR.setDescription("SDI Indexer. " +
"Manage Centroids layers.");
DESCRIPTOR.setVersion(new Semver("1.0.0"));
BBOX_EVALUATORS.add(new BBOXPathScanner());
BBOX_EVALUATORS.add(new BBOXByCoordinatePaths());
}
@Override
public PluginDescriptor getDescriptor() {
return DESCRIPTOR;
}
@Override
public InitializationReport initInContext() throws InitializationException {
InitializationReport report = new InitializationReport();
report.setStatus(Report.Status.OK);
return report;
}
/**
* Expected parameters :
* - indexName (unique)
* - workspace
* - centroidRecord (OPT)
*
* @param request
* @return
*/
@Override
public IndexDocumentReport index(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Serving Index Request {} ",this.getDescriptor().getId(),request);
Project project =request.getDocument();
UseCaseDescriptor useCaseDescriptor = request.getUseCaseDescriptor();
Document requestArguments=request.getCallParameters();
IndexDocumentReport report= new IndexDocumentReport(request);
try{
// ********* INIT INDEX
// TODO CACHE
PostgisIndexer indexer = getIndexer(useCaseDescriptor,requestArguments);
Document profileConfiguration =getConfigurationFromProfile(useCaseDescriptor).getConfiguration();
log.debug("UseCaseDescriptor Configuration is {} ",profileConfiguration);
// ************* PREPARE RECORD
JSONPathWrapper documentNavigator=new JSONPathWrapper(Serialization.write(project));
Document centroidDoc = new Document();
if(requestArguments.containsKey("centroidRecord"))
centroidDoc.putAll(requestArguments.get("centroidRecords",Document.class));
// DEFAULT VALUES
centroidDoc.put(DBConstants.Defaults.PROJECT_ID, project.getId());
centroidDoc.put(DBConstants.Defaults.DISPLAYED,true);
// ********************** EVALAUTE POSITION
log.debug("indexing UseCaseDescriptor {} : Evaluating Centroid... ", useCaseDescriptor.getId());
SpatialReference reference =null;
List<IdentificationReference> refs=project.getIdentificationReferenceByType(SpatialReference.SPATIAL_REFERENCE_TYPE);
if(!refs.isEmpty()){
// Use existing Reference
reference = Serialization.convert(refs.get(0), SpatialReference.class);
log.debug("Using already defined spatial reference " + reference);
GeoJsonObject object = Serialization.convert(reference.getGeoJson(), GeoJsonObject.class);
GCubeSDILayer.BBOX bbox = GCubeSDILayer.BBOX.fromGeoJSON(object.getBbox());
log.info("Found declared BBOX {} ", bbox);
Double pointX = (bbox.getMaxX() + bbox.getMinX())/2;
Double pointY = (bbox.getMaxY() + bbox.getMinY())/2;
String wkt = String.format("POINT (%1$f %2$f) ",
pointX, pointY);
centroidDoc.put("geom", wkt);
} else{
// unable to use current Spatial reference, try evaluating it
log.debug("UseCaseDescriptor {} : Getting evaluation paths from useCaseDescriptor.. ", useCaseDescriptor.getId());
// for each configuration option try until found
GCubeSDILayer.BBOX toSet = null;
for(BBOXEvaluator evaluator : BBOX_EVALUATORS){
log.trace("UCD {}, Project {}. Evaluating BBOX with {}",useCaseDescriptor.getId(),project.getId(),evaluator);
try{
if(evaluator.isConfigured(profileConfiguration)){
toSet=evaluator.evaluate(profileConfiguration,useCaseDescriptor,documentNavigator);
if(toSet!=null) {
log.info("UCD {}, Project {}. Evaluated BBOX {} with method {}",
useCaseDescriptor.getId(),project.getId(),toSet,evaluator);
break;
}
}
}catch (Throwable t){
log.warn("UCD {}, Project {}. Exception with {}",
useCaseDescriptor.getId(),project.getId(),evaluator,t);
}
}
if(toSet== null)
throw new IndexingException("No BBOX has been evaluated from project");
Double pointX=(toSet.getMaxX()+toSet.getMinX())/2;
Double pointY = (toSet.getMaxY()+toSet.getMinY())/2;
log.info("Evaluated BBOX {} ",toSet);
String wkt = String .format("POINT (%1$f %2$f) ",
pointX, pointY);
//TODO support altitude
Double pointZ= 0d;
centroidDoc.put("geom",wkt);
Point point = new Point();
point.setCoordinates(new LngLatAlt(pointX,pointY,pointZ));
point.setBbox(toSet.asGeoJSONArray());
//TODO Manage CRS
point.setCrs(new Crs());
reference = new SpatialReference(Serialization.asDocument(point));
log.info("UCD {} project {}, Setting Spatial Reference {} ",useCaseDescriptor.getId(),project.getId(),Serialization.write(reference));
report.addIdentificationReference(reference);
}
//*********** Additional Values from useCaseDescriptor
log.info("Setting additional values to centroid from mappings ..");
for(MappingObject m : getMappings(useCaseDescriptor)){
List<Object> foundValues = documentNavigator.getByPath(m.getPath());
Object toSetValue=null;
if(!foundValues.isEmpty()) {
// NB CSV for multiple values
StringBuilder b=new StringBuilder();
foundValues.forEach(o-> {
// Parser returns list of list
if (o instanceof Collection) ((Collection<?>) o).forEach(v ->b.append(v + ","));
else b.append(o+",");
});
b.deleteCharAt(b.length()-1);
toSetValue = b.toString();
}
log.trace("Setting {} = {} in centroid doc ",m.getName(),toSetValue);
centroidDoc.put(m.getName(),toSetValue);
}
log.info("Inserting Centroid {} into {} ",Serialization.write(centroidDoc.toJson()),indexer);
indexer.insert(centroidDoc);
// Support to HIDE AND DISPLAY as requested by invoker
if(requestArguments.containsKey("_toHideIds")){
List<String> ids = Serialization.convert(requestArguments.get("_toHideIds"),List.class);
log.info("Requested to hide centroids {} ",ids);
indexer.updateIsVisible(false,ids);
}
if(requestArguments.containsKey("_toDisplayIds")){
List<String> ids = Serialization.convert(requestArguments.get("_toDisplayIds"),List.class);
log.info("Requested to display centroids {} ",ids);
indexer.updateIsVisible(true,ids);
}
report.setStatus(Report.Status.OK);
}catch (SDIInteractionException e){
log.error("Unable to index "+request,e);
report.setStatus(Report.Status.ERROR);
report.putMessage(e.getMessage());
}catch (Throwable t){
log.error("Unable to index "+request,t);
report.setStatus(Report.Status.ERROR);
report.putMessage(t.getMessage());
}finally{
return report;
}
}
private List<MappingObject> getMappings(UseCaseDescriptor useCaseDescriptor) throws InvalidProfileException {
return MappingObject.getMappingsFromUCD(useCaseDescriptor, getDescriptor().getId());
}
@Override
public IndexDocumentReport deindex(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Serving Index Request {} ",this.getDescriptor().getId(),request);
IndexDocumentReport report= new IndexDocumentReport(request);
try{
PostgisIndexer indexer = getIndexer(request.getUseCaseDescriptor(),request.getCallParameters());
indexer.removeByFieldValue(PostgisIndexer.StandardFields.PROJECT_ID,request.getDocument().getId());
}catch (SDIInteractionException e){
log.error("Unable to index "+request,e);
report.setStatus(Report.Status.ERROR);
report.putMessage(e.getMessage());
}catch (Throwable t){
log.error("Unable to index "+request,t);
report.setStatus(Report.Status.ERROR);
report.putMessage(t.getMessage());
}finally{
return report;
}
}
/**
* Expected parameters :
* workspace
* indexName
*
* @param request
* @return
* @throws ConfigurationException
*/
@Override
public Index getIndex(BaseRequest request) throws ConfigurationException {
try {
return getIndexer(request.getUseCaseDescriptor(), request.getCallParameters()).getIndexConfiguration();
}catch(Throwable t ){
throw new ConfigurationException("Unable to get Postgis index for ucd "+request.getUseCaseDescriptor().getId()+" in "+ request.getContext(),t);
}
}
// Inits index
// TODO CACHE
private PostgisIndexer getIndexer(UseCaseDescriptor ucd,Document params) throws ConfigurationException, SQLException, InvalidProfileException, SDIInteractionException {
PostgisIndexer indexer = new PostgisIndexer(sdiCache.getObject(), ucd, postgisCache.getObject());
List<MappingObject> mappingObjects = getMappings(ucd);
List<PostgisTable.Field> fields = PostgisTable.Field.fromMappings(mappingObjects);
indexer.initIndex(params.getString("indexName"),
fields,
params.getString("workspace"),
params.getString("indexName"));
return indexer;
}
static final PluginDescriptor DESCRIPTOR = new PluginDescriptor(Constants.INDEXER_PLUGIN_ID,
IndexerPluginDescriptor.INDEXER);
static final ArrayList<BBOXEvaluator> BBOX_EVALUATORS = new ArrayList<>();
static {
DESCRIPTOR.setDescription("SDI Indexer. " + "Manage Centroids layers.");
DESCRIPTOR.setVersion(new Semver("1.0.0"));
BBOX_EVALUATORS.add(new BBOXPathScanner());
BBOX_EVALUATORS.add(new BBOXByCoordinatePaths());
}
@Override
public PluginDescriptor getDescriptor() {
return DESCRIPTOR;
}
@Override
public InitializationReport initInContext() throws InitializationException {
InitializationReport report = new InitializationReport();
report.setStatus(Report.Status.OK);
return report;
}
/**
* Expected parameters : - indexName (unique) - workspace - flagInternalIndex as
* Boolean; boolean - centroidRecord (OPT).
*
* @param request the request
* @return the index document report
* @throws InvalidPluginRequestException the invalid plugin request exception
*/
@Override
public IndexDocumentReport index(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Serving Index Request {} ", this.getDescriptor().getId(), request);
Project project = request.getDocument();
UseCaseDescriptor useCaseDescriptor = request.getUseCaseDescriptor();
Document requestArguments = request.getCallParameters();
log.debug("requestArguments is {} ", requestArguments);
IndexDocumentReport report = new IndexDocumentReport(request);
try {
// ********* INIT INDEX
// TODO CACHE
PostgisIndexer indexer = getIndexer(useCaseDescriptor, requestArguments);
Document profileConfiguration = getConfigurationFromProfile(useCaseDescriptor).getConfiguration();
log.debug("UseCaseDescriptor Configuration is {} ", profileConfiguration);
// ************* PREPARE RECORD
JSONPathWrapper documentNavigator = new JSONPathWrapper(Serialization.write(project));
Document centroidDoc = new Document();
if (requestArguments.containsKey("centroidRecord"))
centroidDoc.putAll(requestArguments.get("centroidRecords", Document.class));
// DEFAULT VALUES
centroidDoc.put(DBConstants.Defaults.PROJECT_ID, project.getId());
centroidDoc.put(DBConstants.Defaults.DISPLAYED, true);
// Added by Francesco. Creating Gis Viewer Link as public or private
Boolean isInternalIndex = null;
try {
isInternalIndex = requestArguments
.getBoolean(IndexConstants.INDEX_PARAMETER_FLAGINTERNALINDEX);
log.debug(IndexConstants.INDEX_PARAMETER_FLAGINTERNALINDEX + " read as {} ", isInternalIndex);
} catch (Exception e) {
// TODO: handle exception
}
log.info(IndexConstants.INDEX_PARAMETER_FLAGINTERNALINDEX + " is {} ", isInternalIndex);
if (isInternalIndex != null) {
try {
log.debug("Trying to generate Geoportal Gis Link...");
// Contacting the Geoportal-Resolver via UriResolverManager
UriResolverManager uriResolverManager = new UriResolverManager("GEO");
GeoportalResolverQueryStringBuilder builder = new GeoportalResolverQueryStringBuilder(
project.getProfileID(), project.getId());
builder.scope(request.getContext().getId());
if (isInternalIndex) {
builder.resolverAs(RESOLVE_AS.PRIVATE);
} else {
builder.resolverAs(RESOLVE_AS.PUBLIC);
}
Map<String, String> params = builder.buildQueryParameters();
String shortLink = uriResolverManager.getLink(params, true);
log.info("Geoportal GisViewer link is {} ", shortLink);
centroidDoc.put(DBConstants.Defaults.GEOVIEWER_LINK_FIELD, shortLink);
} catch (Exception e) {
log.error("Error on creating the Geoportal GisViewer link for project id {}", project.getId(), e);
}
}
// ********************** EVALAUTE POSITION
log.debug("indexing UseCaseDescriptor {} : Evaluating Centroid... ", useCaseDescriptor.getId());
SpatialReference reference = null;
List<IdentificationReference> refs = project
.getIdentificationReferenceByType(SpatialReference.SPATIAL_REFERENCE_TYPE);
if (!refs.isEmpty()) {
// Use existing Reference
reference = Serialization.convert(refs.get(0), SpatialReference.class);
log.debug("Using already defined spatial reference " + reference);
GeoJsonObject object = Serialization.convert(reference.getGeoJson(), GeoJsonObject.class);
GCubeSDILayer.BBOX bbox = GCubeSDILayer.BBOX.fromGeoJSON(object.getBbox());
log.info("Found declared BBOX {} ", bbox);
Double pointX = (bbox.getMaxX() + bbox.getMinX()) / 2;
Double pointY = (bbox.getMaxY() + bbox.getMinY()) / 2;
String wkt = String.format("POINT (%1$f %2$f) ", pointX, pointY);
centroidDoc.put("geom", wkt);
} else {
// unable to use current Spatial reference, try evaluating it
log.debug("UseCaseDescriptor {} : Getting evaluation paths from useCaseDescriptor.. ",
useCaseDescriptor.getId());
// for each configuration option try until found
GCubeSDILayer.BBOX toSet = null;
for (BBOXEvaluator evaluator : BBOX_EVALUATORS) {
log.trace("UCD {}, Project {}. Evaluating BBOX with {}", useCaseDescriptor.getId(), project.getId(),
evaluator);
try {
if (evaluator.isConfigured(profileConfiguration)) {
toSet = evaluator.evaluate(profileConfiguration, useCaseDescriptor, documentNavigator);
if (toSet != null) {
log.info("UCD {}, Project {}. Evaluated BBOX {} with method {}",
useCaseDescriptor.getId(), project.getId(), toSet, evaluator);
break;
}
}
} catch (Throwable t) {
log.warn("UCD {}, Project {}. Exception with {}", useCaseDescriptor.getId(), project.getId(),
evaluator, t);
}
}
if (toSet == null)
throw new IndexingException("No BBOX has been evaluated from project");
Double pointX = (toSet.getMaxX() + toSet.getMinX()) / 2;
Double pointY = (toSet.getMaxY() + toSet.getMinY()) / 2;
log.info("Evaluated BBOX {} ", toSet);
String wkt = String.format("POINT (%1$f %2$f) ", pointX, pointY);
// TODO support altitude
Double pointZ = 0d;
centroidDoc.put("geom", wkt);
Point point = new Point();
point.setCoordinates(new LngLatAlt(pointX, pointY, pointZ));
point.setBbox(toSet.asGeoJSONArray());
// TODO Manage CRS
point.setCrs(new Crs());
reference = new SpatialReference(Serialization.asDocument(point));
log.info("UCD {} project {}, Setting Spatial Reference {} ", useCaseDescriptor.getId(), project.getId(),
Serialization.write(reference));
report.addIdentificationReference(reference);
}
// *********** Additional Values from useCaseDescriptor
log.info("Setting additional values to centroid from mappings ..");
for (MappingObject m : getMappings(useCaseDescriptor)) {
List<Object> foundValues = documentNavigator.getByPath(m.getPath());
Object toSetValue = null;
if (!foundValues.isEmpty()) {
// NB CSV for multiple values
StringBuilder b = new StringBuilder();
foundValues.forEach(o -> {
// Parser returns list of list
if (o instanceof Collection)
((Collection<?>) o).forEach(v -> b.append(v + ","));
else
b.append(o + ",");
});
b.deleteCharAt(b.length() - 1);
toSetValue = b.toString();
}
log.trace("Setting {} = {} in centroid doc ", m.getName(), toSetValue);
centroidDoc.put(m.getName(), toSetValue);
}
log.info("Inserting Centroid {} into {} ", Serialization.write(centroidDoc.toJson()), indexer);
indexer.insert(centroidDoc);
// Support to HIDE AND DISPLAY as requested by invoker
if (requestArguments.containsKey("_toHideIds")) {
List<String> ids = Serialization.convert(requestArguments.get("_toHideIds"), List.class);
log.info("Requested to hide centroids {} ", ids);
indexer.updateIsVisible(false, ids);
}
if (requestArguments.containsKey("_toDisplayIds")) {
List<String> ids = Serialization.convert(requestArguments.get("_toDisplayIds"), List.class);
log.info("Requested to display centroids {} ", ids);
indexer.updateIsVisible(true, ids);
}
report.setStatus(Report.Status.OK);
} catch (SDIInteractionException e) {
log.error("Unable to index " + request, e);
report.setStatus(Report.Status.ERROR);
report.putMessage(e.getMessage());
} catch (Throwable t) {
log.error("Unable to index " + request, t);
report.setStatus(Report.Status.ERROR);
report.putMessage(t.getMessage());
} finally {
return report;
}
}
private List<MappingObject> getMappings(UseCaseDescriptor useCaseDescriptor) throws InvalidProfileException {
return MappingObject.getMappingsFromUCD(useCaseDescriptor, getDescriptor().getId());
}
@Override
public IndexDocumentReport deindex(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Serving Index Request {} ", this.getDescriptor().getId(), request);
IndexDocumentReport report = new IndexDocumentReport(request);
try {
PostgisIndexer indexer = getIndexer(request.getUseCaseDescriptor(), request.getCallParameters());
indexer.removeByFieldValue(PostgisIndexer.StandardFields.PROJECT_ID, request.getDocument().getId());
} catch (SDIInteractionException e) {
log.error("Unable to index " + request, e);
report.setStatus(Report.Status.ERROR);
report.putMessage(e.getMessage());
} catch (Throwable t) {
log.error("Unable to index " + request, t);
report.setStatus(Report.Status.ERROR);
report.putMessage(t.getMessage());
} finally {
return report;
}
}
/**
* Expected parameters : workspace indexName
*
* @param request
* @return
* @throws ConfigurationException
*/
@Override
public Index getIndex(BaseRequest request) throws ConfigurationException {
try {
return getIndexer(request.getUseCaseDescriptor(), request.getCallParameters()).getIndexConfiguration();
} catch (Throwable t) {
throw new ConfigurationException("Unable to get Postgis index for ucd "
+ request.getUseCaseDescriptor().getId() + " in " + request.getContext(), t);
}
}
// Inits index
// TODO CACHE
private PostgisIndexer getIndexer(UseCaseDescriptor ucd, Document params)
throws ConfigurationException, SQLException, InvalidProfileException, SDIInteractionException {
PostgisIndexer indexer = new PostgisIndexer(sdiCache.getObject(), ucd, postgisCache.getObject());
List<MappingObject> mappingObjects = getMappings(ucd);
List<PostgisTable.Field> fields = PostgisTable.Field.fromMappings(mappingObjects);
indexer.initIndex(params.getString(IndexConstants.INDEX_PARAMETER_INDEXNAME), fields,
params.getString(IndexConstants.INDEX_PARAMETER_WORKSPACE),
params.getString(IndexConstants.INDEX_PARAMETER_INDEXNAME));
return indexer;
}
}

19
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java
View File

@ -1,9 +1,10 @@
package org.gcube.application.cms.sdi.plugins;
import com.vdurmont.semver4j.Semver;
import lombok.Data;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.bson.Document;
import org.gcube.application.cms.implementations.utils.UserUtils;
import org.gcube.application.cms.plugins.MaterializationPlugin;
@ -28,12 +29,12 @@ import org.gcube.application.geoportal.common.model.plugins.MaterializerPluginDe
import org.gcube.application.geoportal.common.model.plugins.PluginDescriptor;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.ContextUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.vdurmont.semver4j.Semver;
import lombok.Data;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SDIMaterializerPlugin extends AbstractPlugin implements MaterializationPlugin {