From b68257d150828b670202be27ab666a44e952aff9 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Mon, 2 Aug 2021 17:43:30 +0200 Subject: [PATCH] Purge upon deletion (optional) --- .../geoportal/service/engine/SDIManager.java | 6 +- .../service/engine/WorkspaceManager.java | 4 + .../engine/mongo/ConcessioniMongoManager.java | 148 ++++++++++++++++-- .../internal/faults/DeletionException.java | 23 +++ .../service/rest/ConcessioniOverMongo.java | 6 +- .../service/ConcessioniOverMongoTest.java | 56 ++++--- 6 files changed, 200 insertions(+), 43 deletions(-) create mode 100644 src/main/java/org/gcube/application/geoportal/service/model/internal/faults/DeletionException.java diff --git a/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java b/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java index eb7f386..3aa8d28 100644 --- a/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java +++ b/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java @@ -364,7 +364,11 @@ public class SDIManager { } //delete file - dtGeoServer.getWebClient().delete(toDelete.getGeoserverPath()); + + // TODO REMOVE HARDCODED PATCH + String path=toDelete.getGeoserverPath().replace("/srv/geoserver_data","geoserver"); +// path=toDelete.getGeoserverPath(); + dtGeoServer.getWebClient().delete(path); } diff --git a/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java b/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java index d6969e3..a8ba056 100644 --- a/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java +++ b/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java @@ -69,6 +69,10 @@ public class WorkspaceManager { return sgClient.open(id).asFolder(); } + public void removeFolderById(String id) throws StorageHubException { + sgClient.open(id).asFolder().delete(); + } + public FolderContainer getSubFolder(FolderContainer parentFolder,String path) throws StorageHubException { try{ return parentFolder.openByRelativePath(path).asFolder(); diff --git a/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java b/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java index 8fab57c..5310378 100644 --- a/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java +++ b/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java @@ -18,14 +18,13 @@ import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.application.geoportal.service.engine.WorkspaceManager.FileOptions; import org.gcube.application.geoportal.service.engine.WorkspaceManager.FolderOptions; import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex; -import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException; -import org.gcube.application.geoportal.service.model.internal.faults.InvalidStateException; -import org.gcube.application.geoportal.service.model.internal.faults.PublishException; -import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; +import org.gcube.application.geoportal.service.model.internal.faults.*; import org.gcube.application.geoportal.service.utils.Serialization; import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.data.transfer.library.faults.RemoteServiceException; import java.io.IOException; +import java.net.MalformedURLException; import java.sql.SQLException; import java.time.LocalDateTime; import java.util.ArrayList; @@ -95,8 +94,8 @@ public class ConcessioniMongoManager extends MongoManager{ return asConcessione(replace(asDocument(toReturn),collectionName)); } - - + + public List list(){ ArrayList toReturn=new ArrayList<>(); iterate(null, collectionName).forEach( @@ -107,19 +106,57 @@ public class ConcessioniMongoManager extends MongoManager{ toReturn.add(asConcessione(d)); }catch(Throwable t) { log.error("Unable to read Document as concessione ",t); - log.debug("Document was "+d.toJson()); + log.debug("Document was "+d.toJson()); } } }); return toReturn; } - public Concessione getById(String id) throws JsonProcessingException, IOException { + public Concessione getById(String id)throws IOException { log.debug("Loading by ID "+id); return asConcessione(getById(asId(id),collectionName)); } - public void deleteById(String id) { - delete(asId(id), collectionName); + + public void deleteById(String id,boolean force) throws DeletionException { + log.debug("Deleting by ID {}, force {}",id,force); + try{ + Concessione concessione =unpublish(id); + try{ + // UNPUBLISH + + if (!concessione.getReport().getStatus().equals(ValidationStatus.PASSED)&&!force) + throw new DeletionException("Unable to unpublish "+concessione.getMongo_id()); + //clean WS + + concessione = removeContent(concessione); + + if (!concessione.getReport().getStatus().equals(ValidationStatus.PASSED)&&!force) + throw new DeletionException("Unable to unpublish "+concessione.getMongo_id()); + + delete(asId(id), collectionName); + }catch(DeletionException e) { + //storing updated - partially deleted + replace(asDocument(concessione), collectionName); + throw e; + } + }catch(Throwable t){ + throw new DeletionException("Unable to delete "+id,t); + } + } + + + public Concessione unpublish(String id) throws DeletionException { + try{ + Concessione toReturn=asConcessione(getById(asId(id),collectionName)); + removeFromIndex(toReturn); + log.debug("Removed from centroids "+toReturn.getMongo_id()); + toReturn = unpublish(toReturn); + log.debug("Concessione after unpublishing is "+toReturn); + return asConcessione(replace(asDocument(toReturn),collectionName)); + }catch(Throwable t){ + throw new DeletionException("Unable to unpublish "+id,t); + } } public Concessione publish(String id) throws JsonProcessingException, IOException, InvalidStateException{ @@ -139,6 +176,33 @@ public class ConcessioniMongoManager extends MongoManager{ } + public static Concessione removeContent(Concessione concessione) throws DeletionException { + if(concessione.getFolderId()==null) { + log.debug("No content for " + concessione.getMongo_id()); + return concessione; + } + try { + log.debug("Removing content for " + concessione.getMongo_id()); + WorkspaceManager manager = new WorkspaceManager(); + manager.removeFolderById(concessione.getFolderId()); + + //Removing references from Object + concessione.setFolderId(null); + ArrayList list = new ArrayList<>(); + list.add(concessione.getPosizionamentoScavo()); + list.addAll(concessione.getPianteFineScavo()); + list.addAll(concessione.getImmaginiRappresentative()); + list.addAll(concessione.getGenericContent()); + for (AssociatedContent c : list) { + c.getActualContent().clear(); + } + return concessione; + }catch(Throwable t){ + throw new DeletionException("Unable to delete from WS ",t); + } + + } + public Concessione persistContent(String id, String destinationPath, List files) throws Exception{ log.info("Persisting {} files for path {} in concessione ",files.size(),destinationPath,id); try{ @@ -146,7 +210,7 @@ public class ConcessioniMongoManager extends MongoManager{ WorkspaceManager ws=new WorkspaceManager(); //Check Init Base folder FolderContainer baseFolder=null; - if(c.getFolderId()==null) { + if(c.getFolderId()==null) { String folderName=Files.fixFilename("mConcessione"+"_"+c.getNome()+"_"+Serialization.FULL_FORMATTER.format(LocalDateTime.now())); log.info("Creating folder {} for Concessione ID {} ",folderName,id); FolderContainer folder=ws.createFolder(new FolderOptions(folderName, "Base Folder for "+c.getNome(),null)); @@ -182,7 +246,20 @@ public class ConcessioniMongoManager extends MongoManager{ return record; } - + private static Concessione removeFromIndex(Concessione record) { + log.info("Removing from index {} ",record.getId()); + ValidationReport report= new ValidationReport("Remove From Index Report "); + PostgisIndex index; + try { + index = new PostgisIndex(); + index.removeCentroid(record); + report.addMessage(ValidationStatus.PASSED, "Removed centroid"); + } catch (SDIInteractionException | SQLException | ConfigurationException e) { + log.error("Unable to reove from index {} ",record,e); + report.addMessage(ValidationStatus.WARNING, "Internal error while indexing."); + } + return record; + } @@ -196,7 +273,7 @@ public class ConcessioniMongoManager extends MongoManager{ ValidationReport report=new ValidationReport("Publish report"); try { SDIManager sdiManager=new SDIManager(); - ArrayList list=new ArrayList(); + ArrayList list=new ArrayList(); //Concessione String workspace= sdiManager.createWorkspace("gna_conc_"+conc.getMongo_id()); @@ -231,24 +308,61 @@ public class ConcessioniMongoManager extends MongoManager{ return conc; } + private static final Concessione unpublish(Concessione concessione){ + ValidationReport report=new ValidationReport("Unpublish report"); + try{ + SDIManager sdi=new SDIManager(); + ArrayList list=new ArrayList(); + + list.add(concessione.getPosizionamentoScavo()); + list.addAll(concessione.getPianteFineScavo()); + for(AssociatedContent c:list) { + if(c instanceof LayerConcessione) { + //TODO actually delete + List contents=c.getActualContent(); + List toRemove=new ArrayList<>(); + for(PersistedContent p:contents){ + if(p instanceof GeoServerContent){ + try { + sdi.deleteContent((GeoServerContent) p); + toRemove.add(p); + } catch (MalformedURLException e) { + e.printStackTrace(); + } catch (RemoteServiceException e) { + e.printStackTrace(); + } + } + } + c.getActualContent().removeAll(toRemove); + } + } + }catch(SDIInteractionException e){ + report.addMessage(ValidationStatus.WARNING, "Unable to unpublish layers "+e.getMessage()); + } + concessione.setReport(report); + return concessione; + } + + + private static final void store(AssociatedContent content,List files, WorkspaceManager ws, FolderContainer base) throws Exception { - FolderContainer sectionParent=null; + FolderContainer sectionParent=null; if(content instanceof RelazioneScavo) sectionParent = ws .createFolder(new FolderOptions( "relazione","Relazione di scavo : "+content.getTitolo(),base)); - else if (content instanceof UploadedImage) + else if (content instanceof UploadedImage) sectionParent = ws .createFolder(new FolderOptions( "imgs","Immagini rappresentative : "+content.getTitolo(),base)); else if (content instanceof SDILayerDescriptor) //SDI Section - if(content instanceof LayerConcessione) + if(content instanceof LayerConcessione) sectionParent = ws .createFolder(new FolderOptions( content.getTitolo(),"Layer Concessione : "+content.getTitolo(),ws.getSubFolder(base,"layers"))); else throw new Exception("Invalid SDI Content "+content); - else if (content instanceof OtherContent ) + else if (content instanceof OtherContent ) sectionParent = ws .createFolder(new FolderOptions( content.getTitolo(),"Relazione di scavo : "+content.getTitolo(),ws.getSubFolder(base,"other"))); else throw new Exception("Invalid Content "+content); diff --git a/src/main/java/org/gcube/application/geoportal/service/model/internal/faults/DeletionException.java b/src/main/java/org/gcube/application/geoportal/service/model/internal/faults/DeletionException.java new file mode 100644 index 0000000..bb121e7 --- /dev/null +++ b/src/main/java/org/gcube/application/geoportal/service/model/internal/faults/DeletionException.java @@ -0,0 +1,23 @@ +package org.gcube.application.geoportal.service.model.internal.faults; + +public class DeletionException extends Exception { + + public DeletionException() { + } + + public DeletionException(String message) { + super(message); + } + + public DeletionException(String message, Throwable cause) { + super(message, cause); + } + + public DeletionException(Throwable cause) { + super(cause); + } + + public DeletionException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java b/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java index ed5859f..fb0cf86 100644 --- a/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java +++ b/src/main/java/org/gcube/application/geoportal/service/rest/ConcessioniOverMongo.java @@ -89,12 +89,14 @@ public class ConcessioniOverMongo { @DELETE @Produces(MediaType.APPLICATION_JSON) @Path("{"+InterfaceConstants.Parameters.PROJECT_ID+"}") - public void deleteById(@PathParam(InterfaceConstants.Parameters.PROJECT_ID) String id) { + public void deleteById(@PathParam(InterfaceConstants.Parameters.PROJECT_ID) String id, + @QueryParam("force") Boolean forceOption) { new GuardedMethod () { @Override protected Concessione run() throws Exception, WebApplicationException { + Boolean force=(forceOption!=null)?forceOption:false; ConcessioniMongoManager manager=new ConcessioniMongoManager(); - manager.deleteById(id); + manager.deleteById(id,force); return null; } }.execute(); diff --git a/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java b/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java index dc8f4f2..3d912d6 100644 --- a/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java +++ b/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java @@ -91,7 +91,14 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ public void delete() throws Exception { WebTarget target=target(PATH); Concessione c = get(target); + + + check(target.path(c.getMongo_id()).request(MediaType.APPLICATION_JSON).delete(),null); + + Concessione published=getFullPublished(target); + + check(target.path(published.getMongo_id()).request(MediaType.APPLICATION_JSON).delete(),null); } @@ -145,28 +152,7 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ @Test public void publish() throws Exception { WebTarget target=target(PATH); - Concessione c=TestModel.prepareConcessione(1,2); - - c.setNome("Concessione : publish test"); - - - - // Register new - c=register(target,c); - - //Upload files - upload(target,c.getMongo_id(),Paths.RELAZIONE,"relazione.pdf"); - upload(target,c.getMongo_id(),Paths.POSIZIONAMENTO,"pos.shp","pos.shx"); - - // Clash on workspaces - upload(target,c.getMongo_id(),Paths.piantaByIndex(0),"pianta.shp","pianta.shx"); - upload(target,c.getMongo_id(),Paths.imgByIndex(0),"immagine.png"); - upload(target,c.getMongo_id(),Paths.imgByIndex(1),"immagine2.png"); - - - - // Immagini - Concessione published=publish(target, c); + Concessione published=getFullPublished(target); System.out.println("Published : "+published); System.out.println("Report is : "+published.getReport()); assertNotNull(published.getReport()); @@ -181,7 +167,31 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ assertNotNull(published.getCentroidLong()); } - + private Concessione getFullPublished(WebTarget target) throws Exception { + Concessione c=TestModel.prepareConcessione(1,2); + + c.setNome("Concessione : publish test"); + + + + // Register new + c=register(target,c); + + //Upload files + upload(target,c.getMongo_id(),Paths.RELAZIONE,"relazione.pdf"); + upload(target,c.getMongo_id(),Paths.POSIZIONAMENTO,"pos.shp","pos.shx"); + + // Clash on workspaces + upload(target,c.getMongo_id(),Paths.piantaByIndex(0),"pianta.shp","pianta.shx"); + upload(target,c.getMongo_id(),Paths.imgByIndex(0),"immagine.png"); + upload(target,c.getMongo_id(),Paths.imgByIndex(1),"immagine2.png"); + + + + // Immagini + Concessione published=publish(target, c); + return published; + } }