package org.gcube.application.geoportal.service.engine.mongo; import com.fasterxml.jackson.core.JsonProcessingException; import com.mongodb.client.MongoDatabase; import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.bson.Document; import org.bson.types.ObjectId; import org.gcube.application.geoportal.common.faults.PathException; import org.gcube.application.geoportal.common.faults.StorageException; import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport; import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport.ValidationStatus; import org.gcube.application.geoportal.common.model.rest.QueryRequest; import org.gcube.application.geoportal.common.rest.TempFile; import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.service.engine.ImplementationProvider; import org.gcube.application.geoportal.service.engine.materialization.SDIManager; import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.application.geoportal.service.engine.WorkspaceManager.FileOptions; import org.gcube.application.geoportal.service.engine.WorkspaceManager.FolderOptions; import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex; import org.gcube.application.geoportal.service.model.internal.faults.*; import org.gcube.application.geoportal.service.utils.Serialization; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.data.transfer.library.faults.RemoteServiceException; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.sql.SQLException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Consumer; @Slf4j public class ConcessioniMongoManager extends MongoManager{ public ConcessioniMongoManager() throws ConfigurationException { super(); } private static final String collectionName="legacyConcessioni"; private static final String DB_NAME="gna_dev"; private MongoDatabase db=null; @Override @Synchronized protected MongoDatabase getDatabase() { if(db==null) { db=client.getDatabase(DB_NAME); } return db; } protected static Document asDocument (Concessione c) throws JsonProcessingException { Document toReturn=Document.parse(Serialization.write(c)); if(c.getMongo_id()!=null&&!c.getMongo_id().isEmpty()) toReturn.append(ID, asId(c.getMongo_id())); return toReturn; } protected static Concessione asConcessione (Document d) throws JsonProcessingException, IOException { return Serialization.read(d.toJson(), Concessione.class); } // *** PUBLIC METHODS public Concessione registerNew(Concessione toRegister) throws IOException { log.trace("Registering {} ",toRegister); toRegister.setDefaults(); ObjectId id=insert(asDocument(toRegister), collectionName); Concessione toReturn=asConcessione(getById(id,collectionName)); toReturn.setMongo_id(asString(id)); return asConcessione(replace(asDocument(toReturn),collectionName)); } public Concessione replace(Concessione toRegister) throws IOException { log.trace("Replacing {} ",toRegister); toRegister.setDefaults(); return asConcessione(replace(asDocument(toRegister),collectionName)); } public Concessione update(String id,String json) throws IOException { log.trace("Updating id {} with {} ",id,json); Concessione toReturn=asConcessione(update(asId(id),asDoc(json),collectionName)); log.debug("Refreshing defaults.."); toReturn.setDefaults(); return asConcessione(replace(asDocument(toReturn),collectionName)); } public Iterable list(){ LinkedBlockingQueue queue=new LinkedBlockingQueue(); iterate(null,null, collectionName).forEach( new Consumer() { @Override public void accept(Document d) { try { queue.put(asConcessione(d)); }catch(Throwable t) { log.error("Unable to read Document as concessione ",t); log.debug("Document was "+d.toJson()); } } }); return queue; } public Iterable search(Document filter){ log.info("Searching concessione for filter {} ",filter); LinkedBlockingQueue queue=new LinkedBlockingQueue(); iterate(filter,null,collectionName).forEach( (Consumer) (Document d)->{try{ queue.put(asConcessione(d)); }catch(Throwable t){log.warn("Unable to translate "+d);}}); log.info("Returned {} elements ",queue.size()); return queue; } public Iterable query(QueryRequest queryRequest){ log.info("Searching concessione for filter {} ",queryRequest); LinkedBlockingQueue queue=new LinkedBlockingQueue(); query(queryRequest,collectionName).forEach( (Consumer) (Document d)->{try{ queue.put(d); }catch(Throwable t){log.warn("Unable to translate "+d);}}); log.info("Returned {} elements ",queue.size()); return queue; } public Concessione getById(String id)throws IOException { log.debug("Loading by ID "+id); return asConcessione(getById(asId(id),collectionName)); } public void deleteById(String id,boolean force) throws DeletionException { log.debug("Deleting by ID {}, force {}",id,force); try{ Concessione concessione =unpublish(id); try{ // UNPUBLISH if (!concessione.getReport().getStatus().equals(ValidationStatus.PASSED)&&!force) throw new DeletionException("Unable to unpublish "+concessione.getMongo_id()); //clean WS concessione = removeContent(concessione); if (!concessione.getReport().getStatus().equals(ValidationStatus.PASSED)&&!force) throw new DeletionException("Unable to unpublish "+concessione.getMongo_id()); delete(asId(id), collectionName); }catch(DeletionException e) { //storing updated - partially deleted replace(asDocument(concessione), collectionName); throw e; } }catch(Throwable t){ throw new DeletionException("Unable to delete "+id,t); } } public Concessione unpublish(String id) throws DeletionException { try{ Concessione toReturn=asConcessione(getById(asId(id),collectionName)); removeFromIndex(toReturn); log.debug("Removed from centroids "+toReturn.getMongo_id()); toReturn = unpublish(toReturn); log.debug("Concessione after unpublishing is "+toReturn); return asConcessione(replace(asDocument(toReturn),collectionName)); }catch(Throwable t){ throw new DeletionException("Unable to unpublish "+id,t); } } public Concessione publish(String id) throws JsonProcessingException, IOException, InvalidStateException{ Concessione toReturn=asConcessione(getById(asId(id),collectionName)); toReturn.setDefaults(); toReturn.validate(); // MATERIALIZE LAYERS toReturn=publish(toReturn); // replace(asDocument(toReturn),collectionName); // CREATE INDEXES toReturn=index(toReturn); // replace(asDocument(toReturn),collectionName); return asConcessione(replace(asDocument(toReturn),collectionName)); } private static Concessione removeContent(Concessione concessione) throws DeletionException { if(concessione.getFolderId()==null) { log.debug("No content for " + concessione.getMongo_id()); return concessione; } try { log.debug("Removing content for " + concessione.getMongo_id()); WorkspaceManager manager = new WorkspaceManager(); manager.removeFolderById(concessione.getFolderId()); //Removing references from Object concessione.setFolderId(null); ArrayList list = new ArrayList<>(); list.add(concessione.getPosizionamentoScavo()); list.addAll(concessione.getPianteFineScavo()); list.addAll(concessione.getImmaginiRappresentative()); list.addAll(concessione.getGenericContent()); for (AssociatedContent c : list) { c.getActualContent().clear(); } return concessione; }catch(Throwable t){ throw new DeletionException("Unable to delete from WS ",t); } } public Concessione unregisterFileset(String id, String toClearPath) throws Exception { log.info("Clearing Fileset at {} for {} ",toClearPath,id); try { WorkspaceManager ws=new WorkspaceManager(); Concessione c = getById(id); AssociatedContent toClearContent=c.getContentByPath(toClearPath); log.debug("Found content {} for path {}",toClearContent,toClearPath); //checking if published content for(PersistedContent persisted : toClearContent.getActualContent()){ if(persisted instanceof GeoServerContent) throw new Exception ("Cannot clear concessione "+id+" at "+toClearContent+", because it is published."); } for(PersistedContent persisted : toClearContent.getActualContent()){ if(persisted instanceof WorkspaceContent) ws.deleteFromWS((WorkspaceContent) persisted); } toClearContent.getActualContent().clear(); log.debug("Updating dafults for {} ",c); c.setDefaults(); return asConcessione(replace(asDocument(c),collectionName)); }catch(Exception e) { throw new Exception("Unable to unregister files.",e); } } public Concessione persistContent(String id, String destinationPath, List files) throws Exception{ log.info("Persisting {} files for path {} in concessione ",files.size(),destinationPath,id); try{ Concessione c = getById(id); StorageUtils storage=ImplementationProvider.get().getStorageProvider().getObject(); WorkspaceManager ws=new WorkspaceManager(); //Check Init Base folder FolderContainer baseFolder=null; if(c.getFolderId()==null) { String folderName=Files.fixFilename("mConcessione"+"_"+c.getNome()+"_"+Serialization.FULL_FORMATTER.format(LocalDateTime.now())); log.info("Creating folder {} for Concessione ID {} ",folderName,id); FolderContainer folder=ws.createFolder(new FolderOptions(folderName, "Base Folder for "+c.getNome(),null)); c.setFolderId(folder.getId()); } log.debug("Folder id is : "+c.getFolderId()); baseFolder=ws.getFolderById(c.getFolderId()); AssociatedContent section=c.getContentByPath(destinationPath); log.debug("Found section {} for path {}",section,destinationPath); store(section,files,ws,storage,baseFolder); log.debug("Updating dafults for {} ",c); c.setDefaults(); return asConcessione(replace(asDocument(c),collectionName)); }catch(Exception e) { throw new Exception("Unable to save file.",e); } } private static Concessione index(Concessione record) { log.info("Indexing {} ",record.getId()); ValidationReport report= new ValidationReport("Index Report "); PostgisIndex index; try { index = new PostgisIndex(); index.registerCentroid(record); report.addMessage(ValidationStatus.PASSED, "Registered centroid"); } catch (SDIInteractionException | PublishException | SQLException | ConfigurationException e) { log.error("Unable to index {} ",record,e); report.addMessage(ValidationStatus.WARNING, "Internal error while indexing."); } return record; } private static Concessione removeFromIndex(Concessione record) { log.info("Removing from index {} ",record.getMongo_id()); ValidationReport report= new ValidationReport("Remove From Index Report "); PostgisIndex index; try { index = new PostgisIndex(); index.removeCentroid(record); report.addMessage(ValidationStatus.PASSED, "Removed centroid"); } catch (SDIInteractionException | SQLException | ConfigurationException e) { log.error("Unable to remove from index {} ",record,e); report.addMessage(ValidationStatus.WARNING, "Internal error while removing from index."); } return record; } private static Concessione publish(Concessione conc) { // CHECK CONDITION BY PROFILE log.debug("Publishing "+conc.getNome()); ValidationReport report=new ValidationReport("Publish report"); try { SDIManager sdiManager=new SDIManager(); ArrayList list=new ArrayList(); //Concessione String workspace= sdiManager.createWorkspace("gna_conc_"+conc.getMongo_id()); list.add(conc.getPosizionamentoScavo()); list.addAll(conc.getPianteFineScavo()); for(AssociatedContent c:list) { if(c instanceof LayerConcessione) { try { List p=c.getActualContent(); GeoServerContent geoserverPersisted=sdiManager.pushShapeLayerFileSet((SDILayerDescriptor)c, workspace, conc.getMongo_id()); // geoserverPersisted.setAssociated(c); p.add(geoserverPersisted); c.setActualContent(p); }catch(SDIInteractionException e) { log.warn("Unable to publish layers.",e); report.addMessage(ValidationStatus.WARNING, "Layer "+c.getTitolo()+" non pubblicato."); } report.addMessage(ValidationStatus.PASSED, "Pubblicato layer "+c.getTitolo()); } } } catch (SDIInteractionException e1) { report.addMessage(ValidationStatus.WARNING, "Unable to publish layers "+e1.getMessage()); } conc.setReport(report); return conc; } private static final Concessione unpublish(Concessione concessione){ ValidationReport report=new ValidationReport("Unpublish report"); try{ SDIManager sdi=new SDIManager(); ArrayList list=new ArrayList(); list.add(concessione.getPosizionamentoScavo()); list.addAll(concessione.getPianteFineScavo()); for(AssociatedContent c:list) { if(c instanceof LayerConcessione) { List contents=c.getActualContent(); List toRemove=new ArrayList<>(); for(PersistedContent p:contents){ if(p instanceof GeoServerContent){ try { sdi.deleteContent((GeoServerContent) p); toRemove.add(p); } catch (Throwable t) { report.addMessage(ValidationStatus.WARNING,"Cannot delete "+((GeoServerContent) p).getFeatureType()); } } } // Remove GIS coordinates ((LayerConcessione) c).setLayerID(null); ((LayerConcessione) c).setBbox(null); ((LayerConcessione) c).setWmsLink(null); ((LayerConcessione) c).setWorkspace(null); //Remove reference to removed content c.getActualContent().removeAll(toRemove); } } concessione.setCentroidLat(null); concessione.setCentroidLong(null); }catch(SDIInteractionException e){ report.addMessage(ValidationStatus.WARNING, "Unable to unpublish layers "+e.getMessage()); } concessione.setReport(report); return concessione; } private static final void store(AssociatedContent content,List files, WorkspaceManager ws, StorageUtils storage, FolderContainer base) throws StorageHubException, StorageException, PathException { FolderContainer sectionParent=null; if(content instanceof RelazioneScavo) sectionParent = ws .createFolder(new FolderOptions( "relazione","Relazione di scavo : "+content.getTitolo(),base)); else if(content instanceof AbstractRelazione) sectionParent = ws .createFolder(new FolderOptions( "abstract_relazione","Abstract relazione di scavo : "+content.getTitolo(),base)); else if (content instanceof UploadedImage) sectionParent = ws .createFolder(new FolderOptions( "imgs","Immagini rappresentative : "+content.getTitolo(),base)); else if (content instanceof SDILayerDescriptor) //SDI Section if(content instanceof LayerConcessione) sectionParent = ws .createFolder(new FolderOptions( content.getTitolo(),"Layer Concessione : "+content.getTitolo(),ws.getSubFolder(base,"layers"))); else throw new PathException("Invalid SDI Content "+content); else if (content instanceof OtherContent ) sectionParent = ws .createFolder(new FolderOptions( content.getTitolo(),"Relazione di scavo : "+content.getTitolo(),ws.getSubFolder(base,"other"))); else throw new PathException("Invalid Content "+content); content.setActualContent(new ArrayList()); for(TempFile f : files) { InputStream is=null; try{ log.debug("Opening temp file {}",f); String fileUrl=storage.getURL(f.getId()); log.debug("Got URL {} from ID {}",fileUrl,f.getId()); is=new URL(fileUrl).openStream(); WorkspaceContent wsContent=ws.storeToWS(new FileOptions(f.getFilename(),is, "Imported via GeoPortal", sectionParent)); log.debug("Registered "+wsContent+" for "+content); content.getActualContent().add(wsContent); }catch(StorageHubException | IOException e){ throw new StorageException("Unable to store "+f,e); }finally{ if(is!=null) IOUtils.closeQuietly(is); } } content.setMongo_id(asString(new ObjectId())); } }