gCubeSystem
/
gcube-cms-suite
15
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

SDI Plugins delete operations

This commit is contained in:
Fabio Sinibaldi 2022-03-30 12:57:14 +02:00
parent 3d6f86ef91
commit 3ae21711f9
9 changed files with 204 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/GCubeSDILayer.java → geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/sdi/GCubeSDILayer.java
View File

@ -1,13 +1,14 @@
package org.gcube.application.geoportal.common.model.document.filesets;
package org.gcube.application.geoportal.common.model.document.filesets.sdi;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.bson.Document;
import org.gcube.application.geoportal.common.model.document.filesets.Materialization;
import java.util.List;
public class GCubeSDILayer extends Materialization{
public class GCubeSDILayer extends Materialization {
@Data
@NoArgsConstructor

29
geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/sdi/GeoServerPlatform.java Normal file
View File

@ -0,0 +1,29 @@
package org.gcube.application.geoportal.common.model.document.filesets.sdi;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.List;
public class GeoServerPlatform extends PlatformInfo{
public static final String GS_PLATFORM="Geoserver";
public static final String WORKSPACE= "workspace";
public static final String LAYER_NAME= " layerName";
public static final String PERSISTENCE_PATH = "persistencePath";
public static final String FILES="files";
public static final String STORENAME="storeName";
@JsonIgnore
public String getWorkspace(){return this.getString(WORKSPACE);}
@JsonIgnore
public String getLayerName(){return this.getString(LAYER_NAME);}
@JsonIgnore
public String getPersistencePath(){return this.getString(PERSISTENCE_PATH);}
@JsonIgnore
public List getFiles(){return this.get(FILES,List.class);}
@JsonIgnore
public String getStoreName(){return this.getString(STORENAME);}
}

22
geoportal-common/src/main/java/org/gcube/application/geoportal/common/model/document/filesets/sdi/PlatformInfo.java Normal file
View File

@ -0,0 +1,22 @@
package org.gcube.application.geoportal.common.model.document.filesets.sdi;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.bson.Document;
public class PlatformInfo extends Document {
public static final String TYPE="_type";
public static final String HOST="_host";
public static final String ENGINE_VERSION = "_engineVersion";
@JsonIgnore
public String getType(){ return this.getString(TYPE); }
@JsonIgnore
public String getHost(){ return this.getString(HOST); }
@JsonIgnore
public String getEngineVersion(){ return this.getString(ENGINE_VERSION); }
}

7
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/PostgisIndexer.java
View File

@ -3,12 +3,10 @@ package org.gcube.application.cms.sdi.engine;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.plugins.requests.BaseRequest;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.CrossReferencedLayer;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.legacy.SDILayerDescriptor;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
@ -114,7 +112,7 @@ public class PostgisIndexer {
}
}
public void deleteByStringValue(PostgisTable.Field field, String id) throws SDIInteractionException {
public void removeByFieldValue(PostgisTable.Field field, String id) throws SDIInteractionException {
log.info("Deleting {}={} from index {}",field.getName(), id,table.getTablename());
try {
dbManager.deleteByFieldValue(table,field,id);
@ -125,5 +123,4 @@ public class PostgisIndexer {
}
}

52
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/engine/SDIManagerWrapper.java
View File

@ -8,21 +8,21 @@ import it.geosolutions.geoserver.rest.encoder.GSLayerEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.caches.AbstractScopedMap;
import org.gcube.application.cms.plugins.requests.BaseExecutionRequest;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFile;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GeoServerPlatform;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.PlatformInfo;
import org.gcube.application.geoportal.common.model.rest.DatabaseConnection;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import sun.reflect.generics.scope.AbstractScope;
import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor;
import java.net.MalformedURLException;
import java.net.URL;
@ -174,6 +174,50 @@ public class SDIManagerWrapper extends SDIManager{
}
public void deleteLayer(GCubeSDILayer toDelete) throws SDIInteractionException {
log.trace("Deleting {}",toDelete);
try{
AbstractGeoServerDescriptor gs=getCurrentGeoserver();
GeoServerRESTPublisher publisher = gs.getPublisher();
for(Object platformObj : toDelete.getPlatformInfo()){
PlatformInfo info =Serialization.convert(platformObj, PlatformInfo.class);
switch(info.getType()){
case GeoServerPlatform.GS_PLATFORM:{
GeoServerPlatform gsInfo = Serialization.convert(info,GeoServerPlatform.class);
log.trace("Deleting {} ",gsInfo);
// remove store (recursion deletes related layers)
log.trace("Removing datastore {}:{}",gsInfo.getWorkspace(),gsInfo.getStoreName());
if(!publisher.removeDatastore(gsInfo.getWorkspace(),gsInfo.getStoreName(),true))
throw new SDIInteractionException("Unable to remove store "+gsInfo.getWorkspace()+":"+gsInfo.getStoreName());
// remove ws if empty
log.trace("Checking if empty WS {}",gsInfo.getWorkspace());
if(gs.getReader().getDatastores(gsInfo.getWorkspace()).isEmpty())
if(!publisher.removeWorkspace(gsInfo.getWorkspace(),true))
throw new SDIInteractionException("Unable to remove WS "+gsInfo.getWorkspace());
// remove actual files data
// TODO REMOVE HARDCODED PATCH
String path=gsInfo.getPersistencePath().replace("/srv/geoserver_data","geoserver");
log.info("Deleting files at {} [{}]",path,gsInfo.getPersistencePath());
getDtGeoServer().getWebClient().delete(path);
break;
}
default : {
throw new SDIInteractionException("Unable to manage platform "+info);
}
}
}
}catch(SDIInteractionException e){
throw e;
}catch (Throwable t){
throw new SDIInteractionException("Unexpected exception while trying to materialize File Set "+t.getMessage(),t);
}
}
public GCubeSDILayer configureCentroidLayer(String name, String workspace, String storeName, PostgisTable table, DatabaseConnection connection) throws SDIInteractionException {
GCubeSDILayerBuilder builder = new GCubeSDILayerBuilder()

33
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/model/GCubeSDILayerBuilder.java
View File

@ -1,13 +1,12 @@
package org.gcube.application.cms.sdi.model;
import freemarker.core.PlainTextOutputFormat;
import lombok.Getter;
import org.bson.Document;
import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.Materialization;
import sun.misc.GC;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GeoServerPlatform;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.PlatformInfo;
import javax.print.Doc;
import java.util.*;
public class GCubeSDILayerBuilder {
@ -24,13 +23,10 @@ public class GCubeSDILayerBuilder {
public static final String GS_PLATFORM="Geoserver";
GCubeSDILayer theObject = new GCubeSDILayer();
@Getter
Document platformInfo= new Document(Materialization.TYPE,GS_PLATFORM);
GeoServerPlatform platformInfo= new GeoServerPlatform();
@Getter
GCubeSDILayer.BBOX bbox = GCubeSDILayer.BBOX.WORLD;
@ -39,6 +35,7 @@ public class GCubeSDILayerBuilder {
Map<OGC_TYPE,Document> ogcLinks = new HashMap<>();
public GCubeSDILayerBuilder(){
platformInfo.put(PlatformInfo.TYPE,GeoServerPlatform.GS_PLATFORM);
}
@ -54,37 +51,37 @@ public class GCubeSDILayerBuilder {
// @@@@@@@@@@@@@@@@@@ Platform info
public GCubeSDILayerBuilder setWorkspace(String ws){
platformInfo.put("workspace",ws);
platformInfo.put(GeoServerPlatform.WORKSPACE,ws);
return this;
}
public GCubeSDILayerBuilder setHost(String ws){
platformInfo.put("host",ws);
platformInfo.put(PlatformInfo.HOST,ws);
return this;
}
public GCubeSDILayerBuilder setEngineVersion(String ws){
platformInfo.put("engineVersion",ws);
platformInfo.put(PlatformInfo.ENGINE_VERSION,ws);
return this;
}
public GCubeSDILayerBuilder setLayerName(String ws){
platformInfo.put("layerName",ws);
platformInfo.put(GeoServerPlatform.LAYER_NAME,ws);
return this;
}
public GCubeSDILayerBuilder setPersistencePath(String ws){
platformInfo.put("persistencePath",ws);
platformInfo.put(GeoServerPlatform.PERSISTENCE_PATH,ws);
return this;
}
public GCubeSDILayerBuilder setStoreName(String ws){
platformInfo.put("storeName",ws);
platformInfo.put(GeoServerPlatform.STORENAME,ws);
return this;
}
public GCubeSDILayerBuilder setFiles(List<String> ws){
platformInfo.put("files",ws);
platformInfo.put(GeoServerPlatform.FILES,ws);
return this;
}
@ -94,9 +91,9 @@ public class GCubeSDILayerBuilder {
addLink(OGC_TYPE.wms,String.format("https://%1$s/geoserver/%2$s/wms?"
+ "service=WMS&version=1.1.0&request=GetMap&layers=%2$s:%3$s&"
+ "styles=&bbox=%4$f,%5$f,%6$f,%7$f&srs=%8$s&format=application/openlayers&width=%9$d&height=%10$d",
platformInfo.get("host"),
platformInfo.get("workspace"),
platformInfo.get("layerName"),
platformInfo.getHost(),
platformInfo.getWorkspace(),
platformInfo.getLayerName(),
bbox.getMinX(),
bbox.getMinY(),
bbox.getMaxX(),

34
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIIndexerPlugin.java
View File

@ -22,7 +22,7 @@ import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.geojson.GeoJsonObject;
@ -86,7 +86,7 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
@Override
public IndexDocumentReport index(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Performing {} ",this.getDescriptor().getId(),request);
log.info("Indexer {} : Serving Index Request {} ",this.getDescriptor().getId(),request);
Project project =request.getDocument();
UseCaseDescriptor useCaseDescriptor = request.getUseCaseDescriptor();
@ -215,6 +215,26 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
}
}
@Override
public IndexDocumentReport deindex(IndexDocumentRequest request) throws InvalidPluginRequestException {
log.info("Indexer {} : Serving Index Request {} ",this.getDescriptor().getId(),request);
IndexDocumentReport report= new IndexDocumentReport(request);
try{
PostgisIndexer indexer = getIndexer(request.getUseCaseDescriptor(),request.getCallParameters());
indexer.removeByFieldValue(Fields.PROJECT_ID,request.getDocument().getId());
}catch (SDIInteractionException e){
log.error("Unable to index "+request,e);
report.setStatus(Report.Status.ERROR);
report.putMessage(e.getMessage());
}catch (Throwable t){
log.error("Unable to index "+request,t);
report.setStatus(Report.Status.ERROR);
report.putMessage(t.getMessage());
}finally{
return report;
}
}
/**
* Expected parameters :
* workspace
@ -250,11 +270,17 @@ public class SDIIndexerPlugin extends SDIAbstractPlugin implements IndexerPlugin
}
private static class Fields{
public static final PostgisTable.Field PROJECT_ID= new PostgisTable.Field("projectid", PostgisTable.FieldType.TEXT);
public static final PostgisTable.Field GEOM= new PostgisTable.Field("geom", PostgisTable.FieldType.GEOMETRY);
}
private List<PostgisTable.Field> getFields(List<MappingObject> mappings){
List<PostgisTable.Field> fields = new ArrayList<>(); // TODO From UseCaseDescriptor
fields.add(new PostgisTable.Field("geom", PostgisTable.FieldType.GEOMETRY));
fields.add(new PostgisTable.Field("projectid", PostgisTable.FieldType.TEXT));
fields.add(Fields.GEOM);
fields.add(Fields.PROJECT_ID);
mappings.forEach(m -> {
fields.add(new PostgisTable.Field(m.getName(), PostgisTable.FieldType.valueOf(m.getType())));

58
sdi-plugins/src/main/java/org/gcube/application/cms/sdi/plugins/SDIMaterializerPlugin.java
View File

@ -4,28 +4,31 @@ import lombok.Data;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.plugins.implementations.AbstractPlugin;
import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException;
import org.gcube.application.cms.sdi.engine.SDIManagerWrapper;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.plugins.MaterializationPlugin;
import org.gcube.application.cms.plugins.faults.InitializationException;
import org.gcube.application.cms.plugins.faults.InvalidPluginRequestException;
import org.gcube.application.cms.plugins.faults.MaterializationException;
import org.gcube.application.cms.plugins.faults.ShutDownException;
import org.gcube.application.cms.plugins.implementations.AbstractPlugin;
import org.gcube.application.cms.plugins.model.ComparableVersion;
import org.gcube.application.cms.plugins.model.PluginDescriptor;
import org.gcube.application.cms.plugins.reports.InitializationReport;
import org.gcube.application.cms.plugins.reports.MaterializationReport;
import org.gcube.application.cms.plugins.reports.Report;
import org.gcube.application.cms.plugins.requests.MaterializationRequest;
import org.gcube.application.cms.sdi.engine.SDIManagerWrapper;
import org.gcube.application.cms.sdi.faults.SDIInteractionException;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.JSONPathWrapper;
import org.gcube.application.cms.plugins.model.ComparableVersion;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.filesets.Materialization;
import org.gcube.application.geoportal.common.model.document.filesets.RegisteredFileSet;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.ContextUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -153,9 +156,52 @@ public class SDIMaterializerPlugin extends AbstractPlugin implements Materializa
}
}
/**
* Expected parameters
* - fileSetPath
*
* @param request
* @return
* @throws MaterializationException
* @throws InvalidPluginRequestException
*/
@Override
public MaterializationReport dematerialize(MaterializationRequest request) throws MaterializationException, InvalidPluginRequestException {
return null;
log.info("Serving DeMaterialization {} ",request);
MaterializationReport report= new MaterializationReport(request);
try{
SDIManagerWrapper sdi=getSDIManager();
JSONPathWrapper wrapper = new JSONPathWrapper(request.getDocument().getTheDocument().toJson());
for(String s : wrapper.getMatchingPaths(request.getMandatory("fileSetPath"))){
log.debug("Found matching {} ",s);
RegisteredFileSet registeredFileSet=Serialization.convert(wrapper.getByPath(s).get(0),RegisteredFileSet.class);
List<Object> toKeep = new ArrayList<>();
for(Object matObj : registeredFileSet.getMaterializations()){
Materialization mat = Serialization.convert(matObj,Materialization.class);
if(mat.getType().equals(GCubeSDILayer.GCUBE_SDY_LAYER_TYPE)) {
log.debug("Deleting Layer {} ",mat);
sdi.deleteLayer(Serialization.convert(matObj, GCubeSDILayer.class));
}else toKeep.add(matObj);
}
// Resetting remaining materializations
registeredFileSet.put(RegisteredFileSet.MATERIALIZATIONS,toKeep);
// Update FS in doc
wrapper.setElement(s,registeredFileSet);
}
// Resetting Document
report.setResultingDocument(Serialization.asDocument(wrapper.getValueCTX().jsonString()));
}catch (SDIInteractionException e){
log.error("Unable to materialize "+request,e);
report.setStatus(Report.Status.ERROR);
report.putMessage(e.getMessage());
}catch (Throwable t){
log.error("Unable to materialize "+request,t);
report.setStatus(Report.Status.ERROR);
report.putMessage(t.getMessage());
}finally{
return report;
}
}
private static final PluginDescriptor DESCRIPTOR=new PluginDescriptor("SDI-Default-Materializer", PluginDescriptor.BaseTypes.MATERIALIZER);

10
sdi-plugins/src/test/java/org/gcube/application/cms/sdi/plugins/IndexerTest.java
View File

@ -10,24 +10,20 @@ import org.gcube.application.cms.plugins.requests.BaseRequest;
import org.gcube.application.cms.plugins.requests.IndexDocumentRequest;
import org.gcube.application.cms.sdi.engine.PostgisIndexer;
import org.gcube.application.cms.sdi.model.GCubeSDILayerBuilder;
import org.gcube.application.cms.sdi.plugins.SDIIndexerPlugin;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.cms.tests.BasicPluginTest;
import org.gcube.application.cms.tests.TestDocuments;
import org.gcube.application.cms.tests.TestProfiles;
import org.gcube.application.geoportal.common.model.configuration.Index;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.accounting.Context;
import org.gcube.application.geoportal.common.model.document.accounting.User;
import org.gcube.application.geoportal.common.model.document.filesets.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GCubeSDILayer;
import org.gcube.application.geoportal.common.model.document.filesets.Materialization;
import org.gcube.application.geoportal.common.model.document.filesets.sdi.GeoServerPlatform;
import org.gcube.application.geoportal.common.model.rest.ConfigurationException;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import org.gcube.spatial.data.geonetwork.utils.UserUtils;
import org.junit.Test;
import ucar.units.Base;
import static junit.framework.TestCase.*;
import static org.junit.Assume.assumeTrue;
@ -77,7 +73,7 @@ public class IndexerTest extends BasicPluginTest {
for (Object pIObj : layer.getPlatformInfo()){
Document platformDoc = Serialization.asDocument(pIObj);
assertEquals(GCubeSDILayerBuilder.GS_PLATFORM,platformDoc.get(Materialization.TYPE));
assertEquals(GeoServerPlatform.GS_PLATFORM,platformDoc.get(Materialization.TYPE));
}