From 5ea5f1adfda0d595b67ee87491a482a45b74b4cd Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Fri, 1 Oct 2021 17:42:23 +0200 Subject: [PATCH] Fixed incoherence between layer and fileSet base name --- .../clients/StatelessClientTests.java | 64 ++++---- .../geoportal/common/utils/ContextUtils.java | 4 +- .../geoportal/common/utils/Files.java | 47 ++++++ .../geoportal/common/utils/StorageUtils.java | 2 +- .../geoportal/common/FilesTests.java | 19 ++- .../service/engine/WorkspaceManager.java | 10 +- .../{ => materialization}/SDIManager.java | 144 ++++++++++-------- .../engine/mongo/ConcessioniMongoManager.java | 3 +- .../service/engine/postgis/PostgisIndex.java | 2 +- .../model/internal/db/MongoConnection.java | 11 ++ .../service/BasicServiceTestUnit.java | 5 +- .../service/ConcessioniOverMongoTest.java | 74 ++++++--- .../geoportal/service/DescribeGSFolder.java | 43 ++++++ .../geoportal/service/MongoTests.java | 1 + .../geoportal/service/SDITests.java | 48 ------ .../geoportal/service/StorageTests.java | 1 + .../geoportal/service/TokenSetter.java | 35 ----- .../engine/materialization/SDITests.java | 79 ++++++++++ .../service/ws/DescribeWSFolder.java | 14 +- .../service/ws/DisplayWorkspaceTree.java | 15 -- .../service/ws/SetWSFolderVisibility.java | 2 +- .../src/test/resources/logback.xml | 10 +- test-data/concessioni/concessioni04-03.csv | 18 --- .../concessioni/concessioni04-03_filtered.csv | 18 --- .../{immagine.png => immagine1.png} | Bin .../{ => packages}/DATASET_GNA_01.csv | 0 .../{ => packages}/DATASET_GNA_02.csv | 0 .../concessioni/packages/concessioni04-03.csv | 17 +++ .../packages/concessioni04-03_filtered.csv | 1 + .../{ => packages}/concessioni_23_04.csv | 0 .../{ => packages}/invio_08_05.csv | 0 test-data/concessioni/transferTest.json | 1 + use-cases/pom.xml | 21 +++ .../cms/usecases/ClearConcessioni.java | 4 +- .../application/cms/usecases/EditFileSet.java | 2 +- .../application/cms/usecases/Export.java | 2 +- .../application/cms/usecases/Import.java | 0 .../cms/usecases/RepublishAll.java | 7 +- .../cms/usecases/RepublishSingle.java | 3 +- .../cms/usecases/mocks/MockFromFolder.java | 73 ++++----- 40 files changed, 484 insertions(+), 316 deletions(-) rename geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/{ => materialization}/SDIManager.java (83%) create mode 100644 geoportal-service/src/test/java/org/gcube/application/geoportal/service/DescribeGSFolder.java delete mode 100644 geoportal-service/src/test/java/org/gcube/application/geoportal/service/SDITests.java delete mode 100644 geoportal-service/src/test/java/org/gcube/application/geoportal/service/TokenSetter.java create mode 100644 geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java delete mode 100644 geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DisplayWorkspaceTree.java delete mode 100644 test-data/concessioni/concessioni04-03.csv delete mode 100644 test-data/concessioni/concessioni04-03_filtered.csv rename test-data/concessioni/{immagine.png => immagine1.png} (100%) rename test-data/concessioni/{ => packages}/DATASET_GNA_01.csv (100%) rename test-data/concessioni/{ => packages}/DATASET_GNA_02.csv (100%) create mode 100644 test-data/concessioni/packages/concessioni04-03.csv create mode 100644 test-data/concessioni/packages/concessioni04-03_filtered.csv rename test-data/concessioni/{ => packages}/concessioni_23_04.csv (100%) rename test-data/concessioni/{ => packages}/invio_08_05.csv (100%) create mode 100644 test-data/concessioni/transferTest.json rename use-cases/src/{main => test}/java/org/gcube/application/cms/usecases/ClearConcessioni.java (95%) rename use-cases/src/{main => test}/java/org/gcube/application/cms/usecases/EditFileSet.java (97%) rename use-cases/src/{main => test}/java/org/gcube/application/cms/usecases/Export.java (99%) rename use-cases/src/{main => test}/java/org/gcube/application/cms/usecases/Import.java (100%) rename use-cases/src/{main => test}/java/org/gcube/application/cms/usecases/RepublishAll.java (89%) rename use-cases/src/{main => test}/java/org/gcube/application/cms/usecases/RepublishSingle.java (86%) rename use-cases/src/{main => test}/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java (62%) diff --git a/geoportal-client/src/test/java/org/gcube/application/geoportal/clients/StatelessClientTests.java b/geoportal-client/src/test/java/org/gcube/application/geoportal/clients/StatelessClientTests.java index 19536f6..495f853 100644 --- a/geoportal-client/src/test/java/org/gcube/application/geoportal/clients/StatelessClientTests.java +++ b/geoportal-client/src/test/java/org/gcube/application/geoportal/clients/StatelessClientTests.java @@ -107,7 +107,7 @@ public class StatelessClientTests extends BasicVreTests{ @Test public void testUploadFileSet() throws Exception { - Concessione testObject= client.createNew(TestModel.prepareEmptyConcessione()); + Concessione testObject= client.createNew(TestModel.prepareConcessione()); AddSectionToConcessioneRequest request= // FileSets.prepareRequest(new StorageUtils(),Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf")); FileSets.build(Paths.ABSTRACT_RELAZIONE).add( @@ -130,43 +130,24 @@ public class StatelessClientTests extends BasicVreTests{ @Test - public Concessione testPublsh() throws Exception { - Concessione c= client.createNew(TestModel.prepareConcessione(1,1)); - StorageUtils storage = new StorageUtils(); - - String mongoId=c.getMongo_id(); - - client.registerFileSet(mongoId, - FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf"))); - - client.registerFileSet(mongoId, - FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf"))); - - client.registerFileSet(mongoId, - FileSets.prepareRequest(storage,Paths.imgByIndex(0),new File(TestModel.getBaseFolder(),"immagine.png"))); - - client.registerFileSet(mongoId, - FileSets.prepareRequest(storage,Paths.POSIZIONAMENTO,new File(TestModel.getBaseFolder(),"pos.shp"))); - - client.registerFileSet(mongoId, - FileSets.prepareRequest(storage,Paths.piantaByIndex(0),new File(TestModel.getBaseFolder(),"pianta.shp"))); - - - c=client.publish(mongoId); + public void testPublsh() throws Exception { + Concessione c=prepare(); assertTrue(c.getReport().getStatus().equals(ValidationStatus.PASSED)); - return c; + } @Test public void testCleanFileSet() throws Exception { - Concessione testObject=testPublsh(); + Concessione testObject=prepare(); //Precheck to be sure assertFalse(testObject.getPosizionamentoScavo().getActualContent().isEmpty()); assertFalse(testObject.getPianteFineScavo().get(0).getActualContent().isEmpty()); + client.unPublish(testObject.getMongo_id()); + //Clear pos testObject=client.cleanFileSet(testObject.getMongo_id(),Paths.POSIZIONAMENTO); assertTrue(testObject.getPosizionamentoScavo().getActualContent().isEmpty()); @@ -177,6 +158,37 @@ public class StatelessClientTests extends BasicVreTests{ } + private Concessione prepare() throws Exception { + int numImgs=5; + Concessione c= client.createNew(TestModel.prepareConcessione(1,numImgs)); + StorageUtils storage = new StorageUtils(); + + String mongoId=c.getMongo_id(); + + client.registerFileSet(mongoId, + FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf"))); + + client.registerFileSet(mongoId, + FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf"))); + + for(int i=0;i iterator){ diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/ContextUtils.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/ContextUtils.java index 9984053..6adb30a 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/ContextUtils.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/ContextUtils.java @@ -15,7 +15,7 @@ public class ContextUtils { public static String getCurrentScope(){ try{ String token=SecurityTokenProvider.instance.get(); - log.debug("Token is : "+token); + log.debug("Token is : "+token.substring(0,2)+"..."+token.substring(token.length()-3)); if(token==null) throw new Exception("Security Token is null"); AuthorizationEntry entry = authorizationService().get(token); return entry.getContext(); @@ -29,7 +29,7 @@ public class ContextUtils { public static String getCurrentCaller(){ try{ String token=SecurityTokenProvider.instance.get(); - log.debug("Token is : "+token); + log.debug("Token is : "+token.substring(0,2)+"..."+token.substring(token.length()-3)); if(token==null) throw new Exception("Security Token is null"); AuthorizationEntry entry = authorizationService().get(token); return entry.getClientInfo().getId(); diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java index a30245c..45930a6 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java @@ -1,13 +1,60 @@ package org.gcube.application.geoportal.common.utils; +import lombok.extern.slf4j.Slf4j; + import java.io.File; import java.io.IOException; import java.net.URL; import java.nio.charset.Charset; import java.nio.file.Paths; +import java.util.*; +@Slf4j public class Files { + + public static Map> getAllShapeSet(File baseFolder,boolean recursive) throws IOException { + return clusterizeFilesByExtension(baseFolder,".shp",recursive); + } + + /* + Map shpAbsolutePath -> fileset + + */ + private static Map> clusterizeFilesByExtension(File base,String extension,Boolean recursive) throws IOException { + HashMap> toReturn = new HashMap<>(); + log.debug("Clustering "+base.getAbsolutePath()); + List targetFiles=new ArrayList<>(); + + // Identify shps + if(base.isDirectory()){ + // Get all shps + targetFiles.addAll(Arrays.asList(base.listFiles((dir, name)->{return name.endsWith(extension);}))); + // recursive + if(recursive) + for(File f : base.listFiles((dir,name)-> {return new File(dir,name).isDirectory();})) + toReturn.putAll(clusterizeFilesByExtension(f,extension,recursive)); + }else { + targetFiles.add(base); + } + + // Group files by shps + targetFiles.forEach(f->{ + String basename=f.getName().substring(0,f.getName().lastIndexOf(".")); + toReturn.put(f.getAbsolutePath(),getSiblings(f.getParentFile(),basename)); + }); + + return toReturn; + } + + + private static List getSiblings(File location,String baseName){ + List fileset=new ArrayList<>(); + for (File shpSet : location.listFiles((dir, name) -> {return name.startsWith(baseName);})) + fileset.add(shpSet); + return fileset; + } + public static File getFileFromResources(String fileName) { ClassLoader classLoader =Files.class.getClassLoader(); diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java index 8795809..7bc85d4 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java @@ -40,7 +40,7 @@ public class StorageUtils { public TempFile putOntoStorage(File source, String filename) throws RemoteBackendException, FileNotFoundException{ if(source.exists() && source.canRead()) return putOntoStorage(new FileInputStream(source),filename); - else throw new FileNotFoundException("Unable to read "+filename); + else throw new FileNotFoundException("Unable to read "+source.getAbsolutePath()+" ("+filename+")"); } public TempFile[] putOntoStorage(File... source) throws RemoteBackendException, FileNotFoundException{ diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java index d53bc9b..2317dd3 100644 --- a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/FilesTests.java @@ -5,15 +5,20 @@ import org.gcube.application.geoportal.common.utils.Files; import org.junit.Test; import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertTrue; @Slf4j public class FilesTests { + static File baseFolder=new File("../test-data/concessioni"); + @Test public void testNames(){ - for(String name : new File("../test-data/concessioni").list()){ + for(String name : baseFolder.list()){ if(name.contains(".")) { String originalExtension = name.substring(name.indexOf(".")); String obtained = Files.fixFilename(name); @@ -23,4 +28,16 @@ public class FilesTests { } } + @Test + public void testClustering() throws IOException { + Map> map=Files.getAllShapeSet(baseFolder.getParentFile(),true); + map.forEach((k, v)->{ + System.out.println(k); + v.forEach(f->{System.out.println(f.getName());}); + }); + + assertTrue(map.get(new File(baseFolder,"pos.shp").getAbsolutePath()).size()==5); + assertTrue(map.get(new File(baseFolder,"pianta.shp").getAbsolutePath()).size()==8); + } + } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java index 2b280ec..4e1bf75 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java @@ -59,7 +59,7 @@ public class WorkspaceManager { public FolderContainer createFolder(FolderOptions opts) throws StorageHubException { if(opts.getParent()==null) opts.setParent(appBase); - return createFolder(opts,sgClient); + return createFolderRoutine(opts); } public FileContainer getFileById(String id) throws StorageHubException { @@ -88,13 +88,13 @@ public class WorkspaceManager { targetName=path.substring(path.lastIndexOf("/")+1); } log.debug("Creating "+targetName); - return createFolder(new FolderOptions(targetName,"",targetParent),sgClient); + return createFolder(new FolderOptions(targetName,"",targetParent)); } } public WorkspaceContent storeToWS(FileOptions opts) throws FileNotFoundException, StorageHubException { - FileContainer item=createFile(opts,sgClient); + FileContainer item=createFileRoutine(opts); item=sgClient.open(item.getId()).asFile(); WorkspaceContent content=new WorkspaceContent(); @@ -126,13 +126,13 @@ public class WorkspaceManager { } @Synchronized - private static FolderContainer createFolder(FolderOptions opts, StorageHubClient sgClient) throws StorageHubException { + private static FolderContainer createFolderRoutine(FolderOptions opts) throws StorageHubException { opts.setFolderName(Files.fixFilename(opts.getFolderName())); return opts.getParent().newFolder(opts.getFolderName(),opts.getFolderDescription()); } @Synchronized - private static FileContainer createFile(FileOptions opts, StorageHubClient sgClient) throws StorageHubException { + private static FileContainer createFileRoutine(FileOptions opts) throws StorageHubException { opts.setFileName(Files.fixFilename(opts.getFileName())); return opts.getParent().uploadFile(opts.getIs(), opts.getFileName(), opts.getFileDescription()); } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/materialization/SDIManager.java similarity index 83% rename from geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java rename to geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/materialization/SDIManager.java index 5546580..0cdbed6 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/SDIManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/materialization/SDIManager.java @@ -1,4 +1,4 @@ -package org.gcube.application.geoportal.service.engine; +package org.gcube.application.geoportal.service.engine.materialization; import it.geosolutions.geoserver.rest.GeoServerRESTPublisher; import it.geosolutions.geoserver.rest.GeoServerRESTPublisher.UploadMethod; @@ -13,6 +13,7 @@ import lombok.extern.slf4j.Slf4j; import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; import org.gcube.application.geoportal.common.utils.Files; +import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.application.geoportal.service.model.internal.db.PostgisTable; import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; import org.gcube.common.storagehub.client.dsl.FileContainer; @@ -28,6 +29,7 @@ import org.gcube.spatial.data.gis.is.AbstractGeoServerDescriptor; import java.io.File; import java.net.MalformedURLException; import java.net.URL; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -101,75 +103,90 @@ public class SDIManager { log.debug("Publishing "+currentElement+" files to geoserver @ "+geoserverHostName); + if(currentElement.getActualContent()==null||currentElement.getActualContent().isEmpty()) + throw new SDIInteractionException("Nothing to publish"); + GeoServerContent content=new GeoServerContent(); content.setGeoserverHostName(geoserverHostName); content.setWorkspace(workspace); WorkspaceManager wsManager=new WorkspaceManager(); - - - - - currentElement.getActualContent().forEach((PersistedContent c)->{ - try { - if(c instanceof WorkspaceContent) { - WorkspaceContent wc=(WorkspaceContent) c; - FileContainer fc=wsManager.getFileById(wc.getStorageID()); - - String completeFilename=Files.fixFilename(fc.get().getName()); - - - String filename= - completeFilename.contains(".")? - completeFilename.substring(0, completeFilename.indexOf(".")):completeFilename; - - - Destination destination=new Destination(completeFilename); - destination.setCreateSubfolders(true); - destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); - destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); - - destination.setPersistenceId("geoserver"); - destination.setSubFolder("GNA/"+projectId+"/"+ - currentElement.getMongo_id()+"/"+filename); - - log.debug("Sending "+wc+" to "+destination); - TransferResult result=dtGeoServer.httpSource(fc.getPublicLink(), destination); - log.debug("Transferred "+result); - - - content.getFileNames().add(completeFilename); - - content.setGeoserverPath(result.getRemotePath().substring(0,result.getRemotePath().lastIndexOf("/"))); - } - }catch(Throwable t) { - log.warn("Unable to transfer Persisted content"+c,t); + + + // ******** IDENTIFY LAYER NAME correct layer name + // Must be unique under same WS + // equal to shp base name + + String baseName= ""; + + // Chose layer name first identifying filename + for(PersistedContent p:currentElement.getActualContent()){ + if(p instanceof WorkspaceContent) { + WorkspaceContent w= (WorkspaceContent) p; + if(w.getName().endsWith(".shp")) { + log.debug("SHP is {}",w.getName()); + baseName=Files.fixFilename(w.getName().substring(0,w.getName().lastIndexOf('.'))); + break; + } } - - }); - - - - - - - if(content.getFileNames().isEmpty()) - throw new SDIInteractionException("No Persisted content found in "+currentElement); - - String completeFileName=content.getFileNames().get(0); - String filename=completeFileName.contains(".")?completeFileName.substring(0, completeFileName.lastIndexOf(".")):completeFileName; - - String remoteFolder=content.getGeoserverPath(); - - String toSetLayerName=filename; + } + + String toSetLayerName=baseName; + + //Check if layer already exists int count=0; GeoServerRESTReader gsReader=currentGeoserver.getReader(); while(gsReader.getLayer(workspace,toSetLayerName)!=null){ count++; - toSetLayerName=filename+"_"+count; - log.debug("layer for "+filename+" already existing, trying "+toSetLayerName); + toSetLayerName=baseName+"_"+count; + log.debug("layer for "+baseName+" already existing, trying "+toSetLayerName); } + String folderRelativePath="GNA/" + projectId + "/" + + currentElement.getMongo_id() + "/" + toSetLayerName; + + + for (PersistedContent c : currentElement.getActualContent()) { + if (c instanceof WorkspaceContent) { + WorkspaceContent wc = (WorkspaceContent) c; + FileContainer fc = wsManager.getFileById(wc.getStorageID()); + + String completeFilename = Files.fixFilename(fc.get().getName()); + + completeFilename=completeFilename.replaceAll(baseName, toSetLayerName); + + + Destination destination = new Destination(completeFilename); + destination.setCreateSubfolders(true); + destination.setOnExistingFileName(DestinationClashPolicy.REWRITE); + destination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + + destination.setPersistenceId("geoserver"); + destination.setSubFolder(folderRelativePath); + + log.debug("Sending " + wc + " to " + destination); + TransferResult result = SDIManager.this.getDtGeoServer().httpSource(fc.getPublicLink(), destination); + log.debug("Transferred " + result); + + + content.getFileNames().add(completeFilename); + + content.setGeoserverPath(result.getRemotePath().substring(0, result.getRemotePath().lastIndexOf("/"))); + } + } + + + + +// String completeFileName=content.getFileNames().get(0); +// String filename=completeFileName.contains(".")?completeFileName.substring(0, completeFileName.lastIndexOf(".")):completeFileName; +// +// String remoteFolder=content.getGeoserverPath(); + + + + + String storeName=toSetLayerName+"_store"; content.setStore(storeName); content.setFeatureType(toSetLayerName); @@ -177,10 +194,11 @@ public class SDIManager { GeoServerRESTPublisher publisher=currentGeoserver.getPublisher(); log.debug("Trying to create remote workspace : "+workspace); createWorkspace(workspace); - - log.debug("Publishing remote folder "+remoteFolder); - URL directoryPath=new URL("file:"+remoteFolder+"/"+filename+".shp"); + String folderAbsolutePath=content.getGeoserverPath(); + log.debug("Publishing remote folder "+folderAbsolutePath); + + URL directoryPath=new URL("file:"+folderAbsolutePath+"/"+toSetLayerName+".shp"); //TODO Evaluate SRS @@ -410,5 +428,7 @@ public class SDIManager { dtGeoServer.getWebClient().delete(path); } - + + + } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java index eff10a0..8df769a 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java @@ -16,12 +16,11 @@ import org.gcube.application.geoportal.common.rest.TempFile; import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.service.engine.ImplementationProvider; -import org.gcube.application.geoportal.service.engine.SDIManager; +import org.gcube.application.geoportal.service.engine.materialization.SDIManager; import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.application.geoportal.service.engine.WorkspaceManager.FileOptions; import org.gcube.application.geoportal.service.engine.WorkspaceManager.FolderOptions; import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex; -import org.gcube.application.geoportal.service.engine.providers.StorageClientProvider; import org.gcube.application.geoportal.service.model.internal.faults.*; import org.gcube.application.geoportal.service.utils.Serialization; import org.gcube.common.storagehub.client.dsl.FolderContainer; diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java index 9ff150f..514ebb8 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/postgis/PostgisIndex.java @@ -5,7 +5,7 @@ import org.gcube.application.geoportal.common.model.legacy.Concessione; import org.gcube.application.geoportal.common.model.rest.DatabaseConnection; import org.gcube.application.geoportal.common.model.rest.PostgisIndexDescriptor; import org.gcube.application.geoportal.service.engine.ImplementationProvider; -import org.gcube.application.geoportal.service.engine.SDIManager; +import org.gcube.application.geoportal.service.engine.materialization.SDIManager; import org.gcube.application.geoportal.service.model.internal.db.DBConstants; import org.gcube.application.geoportal.service.model.internal.db.PostgisTable; import org.gcube.application.geoportal.service.model.internal.db.PostgisTable.Field; diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/model/internal/db/MongoConnection.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/model/internal/db/MongoConnection.java index 8c81895..30e1509 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/model/internal/db/MongoConnection.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/model/internal/db/MongoConnection.java @@ -13,4 +13,15 @@ public class MongoConnection { private String database; private List hosts=new ArrayList(); private int port; + + @Override + public String toString() { + return "MongoConnection{" + + "user='" + user + '\'' + + ", password=***" + + ", database='" + database + '\'' + + ", hosts=" + hosts + + ", port=" + port + + '}'; + } } diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java index 50d4f15..ed06918 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/BasicServiceTestUnit.java @@ -1,5 +1,7 @@ package org.gcube.application.geoportal.service; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.geoportal.service.rest.GuardedMethod; import org.gcube.application.geoportal.service.utils.Serialization; import org.glassfish.jersey.test.JerseyTest; @@ -7,7 +9,7 @@ import org.junit.BeforeClass; import javax.ws.rs.core.Application; import javax.ws.rs.core.Response; - +@Slf4j public class BasicServiceTestUnit extends JerseyTest { @@ -25,6 +27,7 @@ public class BasicServiceTestUnit extends JerseyTest { GuardedMethod.addPreoperation(new Runnable() { @Override public void run() { + log.debug("TEST IMPL : Setting context "+scope+" in received call"); TokenSetter.set(scope); } }); diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java index b64d5fd..7b76bff 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java @@ -1,5 +1,6 @@ package org.gcube.application.geoportal.service; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.cms.tests.model.TestFilters; import org.gcube.application.cms.tests.model.TestQueries; import org.gcube.application.cms.tests.model.TestModel; @@ -9,6 +10,7 @@ import org.gcube.application.geoportal.common.model.legacy.LayerConcessione; import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport.ValidationStatus; import org.gcube.application.geoportal.common.rest.InterfaceConstants; import org.gcube.application.geoportal.common.utils.FileSets; +import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.service.utils.Serialization; import org.json.JSONObject; @@ -23,6 +25,7 @@ import javax.ws.rs.core.Response; import java.io.File; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import static org.junit.Assert.*; @@ -63,12 +66,21 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ post(Entity.entity(Serialization.write(builder.getTheRequest()), MediaType.APPLICATION_JSON)),Concessione.class); } - - private static Concessione publish(WebTarget target, Concessione conc) throws Exception { - Response resp=target.path(PUBLISH_PATH).path(conc.getMongo_id()).request(MediaType.APPLICATION_JSON). - put(Entity.entity(Serialization.write(conc), MediaType.APPLICATION_JSON)); + + private static Concessione update(Concessione c, WebTarget target) throws Exception { + + return check(target.path(c.getMongo_id()).request(MediaType.APPLICATION_JSON). + put(Entity.entity(Serialization.write(c), + MediaType.APPLICATION_JSON)),Concessione.class); + } + + private static Concessione publish(WebTarget target, String id) throws Exception { + Response resp=target.path(PUBLISH_PATH).path(id).request(MediaType.APPLICATION_JSON). + put(Entity.entity(Serialization.write(id), MediaType.APPLICATION_JSON)); return check(resp,Concessione.class); } + + private static Concessione unpublish(WebTarget target, String id) throws Exception { Response resp=target.path(PUBLISH_PATH).path(id).request(MediaType.APPLICATION_JSON). delete(); @@ -178,10 +190,11 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ @Test public void republish() throws Exception{ WebTarget target=target(PATH); - Concessione published=getFullPublished(target); +// Concessione published=getFullPublished(target); + Concessione published=getById(target,"6155ba6002ad3d2c23b72b5a"); published = unpublish(target,published.getMongo_id()); System.out.println("Republishing.."); - published=publish(target,published); + published=publish(target,published.getMongo_id()); Assert.assertEquals(published.getReport().getStatus(),ValidationStatus.PASSED); } @@ -191,7 +204,7 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ //Republishing WebTarget target=target(PATH); String id="610415af02ad3d05b5f81ee3"; - publish(target,unpublish(target,id)); + publish(target,unpublish(target,id).getMongo_id()); target.path(id).queryParam(InterfaceConstants.Parameters.FORCE,true).request(MediaType.APPLICATION_JSON).delete(); } @@ -279,6 +292,7 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ public void publish() throws Exception { WebTarget target=target(PATH); Concessione published=getFullPublished(target); + System.out.println("Published : "+published); System.out.println("Report is : "+published.getReport()); assertNotNull(published.getReport()); @@ -295,9 +309,14 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ } private Concessione getFullPublished(WebTarget target) throws Exception { - Concessione c=TestModel.prepareConcessione(1,2); - c.setNome("Concessione : publish test"); + File layerFolder=new File( + "/Users/fabioisti/Documents/Concessioni 04-03/UsiniTomestighes"); + Map> layers = Files.getAllShapeSet(layerFolder,true); + + Concessione c=TestModel.prepareConcessione(1,1); + + c.setNome("Concessione : publish test "); StorageUtils storage=new StorageUtils(); @@ -305,27 +324,34 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ c=register(target,c); //Upload files - upload(storage,target,c.getMongo_id(),Paths.RELAZIONE,"relazione.pdf"); - upload(storage,target,c.getMongo_id(),Paths.ABSTRACT_RELAZIONE,"relazione.pdf"); + c=upload(storage,target,c.getMongo_id(),Paths.RELAZIONE,"relazione.pdf"); + c=upload(storage,target,c.getMongo_id(),Paths.ABSTRACT_RELAZIONE,"relazione.pdf"); - upload(storage,target,c.getMongo_id(),Paths.POSIZIONAMENTO, -// TestModel.getBaseFolder().list((file,name)->{return name.startsWith("pos");})); - new File("/Users/fabioisti/Documents/invio_08_05/Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va/CONSEGNA_WGS84") - .listFiles((file,name)->{return name.startsWith("Mag_area");})); - // Clash on workspaces - upload(storage,target,c.getMongo_id(),Paths.piantaByIndex(0), -// TestModel.getBaseFolder().list((file,name)->{return name.startsWith("pianta");})); - new File("/Users/fabioisti/Documents/invio_08_05/Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va/CONSEGNA_WGS84") - .listFiles((file,name)->{return name.startsWith("Mag_anomalies_WGS84");})); - upload(storage,target,c.getMongo_id(),Paths.imgByIndex(0),"immagine.png"); - upload(storage,target,c.getMongo_id(),Paths.imgByIndex(1),"immagine2.png"); + String[] keys=layers.keySet().toArray(new String [0]); + c=upload(storage,target,c.getMongo_id(),Paths.POSIZIONAMENTO, + TestModel.getBaseFolder().list((file,name)->{return name.startsWith("pianta.shp");})); +// layers.get(keys[0]).toArray(new File[0])); + // Clash on workspaces + for(int i=0;i{return name.startsWith("pianta.shp");})); +// layers.get(key).toArray(new File[0])); + } // Immagini - Concessione published=publish(target, c); - return published; + for (int i = 0; i iterator){ diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/DescribeGSFolder.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/DescribeGSFolder.java new file mode 100644 index 0000000..9e5a075 --- /dev/null +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/DescribeGSFolder.java @@ -0,0 +1,43 @@ +package org.gcube.application.geoportal.service; + +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.tests.TokenSetter; +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.faults.RemoteServiceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.model.RemoteFileDescriptor; + + + +@Slf4j +public class DescribeGSFolder { + + public static void main(String[] args) throws UnreachableNodeException, ServiceNotFoundException, RemoteServiceException { + + TokenSetter.set("/gcube/devsec/devVRE"); + + String gsEndpoint="https://geoserver-218.dev.d4science.org/"; + String gsBasePath="geoserver/GNA/6157089e596924b380a174b7_1633101414296/"; + DataTransferClient client=DataTransferClient.getInstanceByEndpoint(gsEndpoint); +// System.out.println(getFileDescriptor(gsBasePath,client.getWebClient())); + print(client,gsBasePath,""); + } + + + public static void print(DataTransferClient client, String path, String pad) throws RemoteServiceException { + RemoteFileDescriptor desc=client.getWebClient().getInfo(path); + System.out.println(pad+desc.getFilename() + "["+desc.getSize()+"]"); + if(desc.isDirectory()) + desc.getChildren().forEach(f->{ + try { + print(client,path+"/"+f,pad+"\t"); + } catch (RemoteServiceException e) { + e.printStackTrace(); + } + }); + + } + + +} diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/MongoTests.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/MongoTests.java index 302fe7c..bdd1c94 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/MongoTests.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/MongoTests.java @@ -7,6 +7,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Projections; import org.bson.Document; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.geoportal.service.engine.ImplementationProvider; import org.gcube.application.geoportal.service.engine.mongo.MongoManager; import org.gcube.application.geoportal.service.engine.providers.MongoClientProvider; diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/SDITests.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/SDITests.java deleted file mode 100644 index c0f81cf..0000000 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/SDITests.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.gcube.application.geoportal.service; - - -import org.gcube.application.geoportal.service.engine.SDIManager; -import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex; -import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException; -import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.sql.SQLException; -import java.util.regex.Matcher; - -public class SDITests { - - - @Before - public void init(){ - TokenSetter.set("/gcube/devsec/devVRE"); - } - - @Test - public void registerCentroidsLayer() throws SDIInteractionException, SQLException, ConfigurationException { - PostgisIndex index=new PostgisIndex(); - } - - @Test - public void testRegexp(){ - Matcher hostMatcher=SDIManager.HOSTNAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); - Assert.assertTrue(hostMatcher.find()); - System.out.println("HOST :\t"+hostMatcher.group()); - - Matcher portMatcher=SDIManager.PORT_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); - Assert.assertTrue(portMatcher.find()); - System.out.println("PORT :\t"+portMatcher.group()); - - Matcher dbMatcher=SDIManager.DB_NAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); - Assert.assertTrue(dbMatcher.find()); - System.out.println("DB :\t"+dbMatcher.group()); - } - - - @Test - public void registerFileSet(){ - - } -} diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/StorageTests.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/StorageTests.java index 7a7e681..3088782 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/StorageTests.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/StorageTests.java @@ -1,5 +1,6 @@ package org.gcube.application.geoportal.service; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.service.engine.ImplementationProvider; import org.gcube.application.geoportal.service.engine.providers.StorageClientProvider; diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/TokenSetter.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/TokenSetter.java deleted file mode 100644 index 783226d..0000000 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/TokenSetter.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.application.geoportal.service; - -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.scope.api.ScopeProvider; - -import java.util.Properties; - -public class TokenSetter { - - - - private static Properties props=new Properties(); - - static{ - try { - props.load(TokenSetter.class.getResourceAsStream("/tokens.properties")); - } catch (Exception e) { - throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION"); - } - } - - - public static void set(String scope){ - try{ - if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope); - SecurityTokenProvider.instance.set(props.getProperty(scope)); - }catch(Exception e){ - e.printStackTrace(System.err); - throw e; - } - ScopeProvider.instance.set(scope); - } - - -} diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java new file mode 100644 index 0000000..c6558f3 --- /dev/null +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/materialization/SDITests.java @@ -0,0 +1,79 @@ +package org.gcube.application.geoportal.service.engine.materialization; + + +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.tests.TokenSetter; +import org.gcube.application.cms.tests.model.TestModel; +import org.gcube.application.geoportal.common.model.legacy.Concessione; +import org.gcube.application.geoportal.common.model.legacy.GeoServerContent; +import org.gcube.application.geoportal.common.model.legacy.LayerConcessione; +import org.gcube.application.geoportal.common.model.legacy.PersistedContent; +import org.gcube.application.geoportal.common.utils.Files; +import org.gcube.application.geoportal.service.engine.materialization.SDIManager; +import org.gcube.application.geoportal.service.engine.postgis.PostgisIndex; +import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException; +import org.gcube.application.geoportal.service.model.internal.faults.SDIInteractionException; +import org.gcube.application.geoportal.service.utils.Serialization; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; + +@Slf4j +public class SDITests { + + + @Before + public void init(){ + TokenSetter.set("/gcube/devsec/devVRE"); + } + + @Test + public void registerCentroidsLayer() throws SDIInteractionException, SQLException, ConfigurationException { + PostgisIndex index=new PostgisIndex(); + } + + @Test + public void testRegexp(){ + Matcher hostMatcher=SDIManager.HOSTNAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); + Assert.assertTrue(hostMatcher.find()); + System.out.println("HOST :\t"+hostMatcher.group()); + + Matcher portMatcher=SDIManager.PORT_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); + Assert.assertTrue(portMatcher.find()); + System.out.println("PORT :\t"+portMatcher.group()); + + Matcher dbMatcher=SDIManager.DB_NAME_PATTERN.matcher("jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/geoserver_dev_db"); + Assert.assertTrue(dbMatcher.find()); + System.out.println("DB :\t"+dbMatcher.group()); + } + + +// @Test + //Disabled because dependant on state + public void registerLayer() throws IOException, SDIInteractionException { + Concessione toTest= Serialization.read(Files.readFileAsString( + new File(TestModel.getBaseFolder(),"transferTest.json").getAbsolutePath(), Charset.defaultCharset()), Concessione.class); + + + SDIManager manager = new SDIManager(); + List toPush=new ArrayList<>(); + toPush.add(toTest.getPosizionamentoScavo()); + toPush.addAll(toTest.getPianteFineScavo()); + + for (LayerConcessione l :toPush){ + GeoServerContent content=manager.pushShapeLayerFileSet(l,toTest.getFolderId(),toTest.getMongo_id()+"_"+System.currentTimeMillis()); + System.out.println("Published "+content); + } + + + + } +} diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java index 351836f..1244c08 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java @@ -1,6 +1,6 @@ package org.gcube.application.geoportal.service.ws; -import org.gcube.application.geoportal.service.TokenSetter; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.model.exceptions.StorageHubException; @@ -13,7 +13,7 @@ public class DescribeWSFolder { public static void main(String[] args) throws StorageHubException { String context="/gcube/devsec/devVRE"; - String folderID="28774602-6423-4870-9afb-f8f4b585b438"; + String folderID="46b376db-32d7-4411-ad04-ca3dadab5f5b"; Boolean recursive = true; TokenSetter.set(context); @@ -25,19 +25,11 @@ public class DescribeWSFolder { System.out.println("Listing... "); print(folder,""); -// for (Item i : folder.list().includeHidden().getItems()) { -// System.out.println("name:" + i.getName() + " type " + i.getPrimaryType()); -// -//// System.out.println("name:" + i.getName() + " is a File?: " + (i instanceof AbstractFileItem)); -//// System.out.println("name:" + i.getName() + " is a folder?: " + (i instanceof FolderItem)); -//// System.out.println("name:" + i.getName() + " is a shared folder?: " + (i instanceof SharedFolder)); -//// System.out.println("name:" + i.getName() + " is a VRE folder?: " + (i instanceof VreFolder)); -// } } private static final void print(FolderContainer folder,String pad) throws StorageHubException { for (Item i : folder.list().includeHidden().getItems()) { - System.out.println(pad+"name:" + i.getName() + " type " + i.getPrimaryType()); + System.out.println(pad+ i.getName() + " [" + i.getPrimaryType()+"]"); if(i instanceof FolderItem){ print(shc.open(i.getId()).asFolder(),pad+"\t"); } diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DisplayWorkspaceTree.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DisplayWorkspaceTree.java deleted file mode 100644 index 8ffc364..0000000 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DisplayWorkspaceTree.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.gcube.application.geoportal.service.ws; - -public class DisplayWorkspaceTree { - - public static void main(String[] args) { - String context="/gcube/devsec/devVRE"; - - String folderId=null; - - - // NB null == - - } - -} diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java index 8c0084b..c6a7682 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/SetWSFolderVisibility.java @@ -1,6 +1,6 @@ package org.gcube.application.geoportal.service.ws; -import org.gcube.application.geoportal.service.TokenSetter; +import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.StorageHubClient; diff --git a/geoportal-service/src/test/resources/logback.xml b/geoportal-service/src/test/resources/logback.xml index 634a1bb..7b7e4ee 100644 --- a/geoportal-service/src/test/resources/logback.xml +++ b/geoportal-service/src/test/resources/logback.xml @@ -3,13 +3,19 @@ - %d{yyyy-MM-dd HH:mm:ss} | %-5p | [%thread] %logger{5}:%L - %msg%n + %d{HH:mm:ss} | %logger{0}:%L - %msg%n - + + + + + + + \ No newline at end of file diff --git a/test-data/concessioni/concessioni04-03.csv b/test-data/concessioni/concessioni04-03.csv deleted file mode 100644 index 0b58aa6..0000000 --- a/test-data/concessioni/concessioni04-03.csv +++ /dev/null @@ -1,18 +0,0 @@ -"Acquacadda_Nuxis_2019_def","Acquacadda_Nuxis_2019_def/D_posizionamento saggi_NUXIS 2019","Acquacadda_Nuxis_2019_def/E_piante fine scavo vettoriali_NUXIS 2019" -"Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)","Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)/GNA_SAN BASILIO_2019/GNA_topografia_San Basilio/POSIZIONAMENTO SAGGI E AREE DELLE INDAGINI","Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)/GNA_SAN BASILIO_2019/GNA_topografia_San Basilio/PLANIMETRIE SAGGI AREE DELLE INDAGINI" -"Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)","Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)/S.Basilio-UniVe_GNA/D.GNA_Posizionamento_San Basilio","Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)/S.Basilio-UniVe_GNA/E-F.GNA_Pianta fine scavo_San Basilio/GNA_Pianta di fine scavo_San Basilio" -"C_F_GNA_Oscurusciuto_2020","C_F_GNA_Oscurusciuto_2020/D-E-F_GNA_topografia_Oscurusciuto/Posizionamento_limiti_saggi_indagati_Oscurusciuto_vettoriale","C_F_GNA_Oscurusciuto_2020/D-E-F_GNA_topografia_Oscurusciuto/Pianta_fine_scavo_Oscurusciuto_2019_vettoriale" -"Cerchiara-Damale_rev","Cerchiara-Damale_rev/D. Posizionamento saggio","Cerchiara-Damale_rev/D. Posizionamento saggio/E. Pianta fine scavo_vettoriale" -"Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)","Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)/IVa_E_Tav.1_Shape Posizionamento area scavo","Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)/IVa_E_Tav.1_Shape Posizionamento area scavo" -"Cervia Vecchia_rev","Cervia Vecchia_rev/D_GNA_topografia_Cervia","Cervia Vecchia_rev/D_GNA_topografia_Cervia" -"Civitavecchia (RM)_loc. Ficoncella_Aquae Tauri_docum. IVa_2019","Civitavecchia (RM)_loc. Ficoncella_Aquae Tauri_docum. IVa_2019/Civitavecchia (RM)_Ficoncella_documentazione fine scavo 2019_IVa/IVa_D_E_F_topografia_Aquae_Tauri/Posizionamento saggi 2019","Civitavecchia (RM)_loc. Ficoncella_Aquae Tauri_docum. IVa_2019/Civitavecchia (RM)_Ficoncella_documentazione fine scavo 2019_IVa/IVa_D_E_F_topografia_Aquae_Tauri/Pianta di fine scavo 2019" -"Frascineto Timpone delle Fave_rev","Frascineto Timpone delle Fave_rev/D. Posizionamento dell'area","Frascineto Timpone delle Fave_rev/D. Posizionamento dell'area" -"GNA_AQUILEIAcomellimoro_2019_rev","GNA_AQUILEIAcomellimoro_2019_rev/GNA_topografia_AQUILEIAcomellimoro_2019/D.Posizionamento_AQU19COM","GNA_AQUILEIAcomellimoro_2019_rev/GNA_topografia_AQUILEIAcomellimoro_2019/E.Planimetria generale_AQU19COM" -"GNA_Poggio Pimperiale_Poggibonsi","GNA_Poggio Pimperiale_Poggibonsi/5591449/GNA_topografia_PoggioImperiale/POSIZIONAMENTO","GNA_Poggio Pimperiale_Poggibonsi/5591449/GNA_topografia_PoggioImperiale/PLANIMETRIE SAGGI/SHAPE A22" -"Jesolo (Ve) - Loc. San Mauro e Torre del Caligo","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/GNA_topografia_Jesolo/GIS_shp","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/GNA_topografia_Jesolo/GIS_shp" -"Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)","Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)/IVa_D_E_F_GNA_topografia_Via Nomentum-Eretum","Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)/IVa_D_E_F_GNA_topografia_Via Nomentum-Eretum" -"Quarto d'Altino -Loc. Fornace","Quarto d'Altino -Loc. Fornace/Cupitò/GNA_Altino_posizionamento_shp","Quarto d'Altino -Loc. Fornace/Cupitò/GNA_Altino_posizionamento_shp" -"Tolfa (RM)_Bufalareccia_documentazione IVa 2019","Tolfa (RM)_Bufalareccia_documentazione IVa 2019/IVa_D_GNA_posizionamento dei limiti in formato vectoriale_Bufalareccia 2019","Tolfa (RM)_Bufalareccia_documentazione IVa 2019/IVa_E_GNA_pianta di fine scavo multipolygon_Bufalareccia 2019" -"UNIME_ LAINO_REV","UNIME_ LAINO_REV/D. Limiti saggi/laino 2019 limiti saggi","UNIME_ LAINO_REV/D. Limiti saggi/laino 2019 limiti saggi" -"UNIME_TORTORA_rev","UNIME_TORTORA_rev/tortora 2019 limiti saggi","UNIME_TORTORA_rev/tortora 2019 pianta fine scavo" -"UsiniTomestighes","UsiniTomestighes/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/D_GNA_Posizionamento_limiti_aree_indagate","UsiniTomestighes/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/E_GNA_Piante_fine_scavo_Tomestighes/Area A_est" \ No newline at end of file diff --git a/test-data/concessioni/concessioni04-03_filtered.csv b/test-data/concessioni/concessioni04-03_filtered.csv deleted file mode 100644 index 0b58aa6..0000000 --- a/test-data/concessioni/concessioni04-03_filtered.csv +++ /dev/null @@ -1,18 +0,0 @@ -"Acquacadda_Nuxis_2019_def","Acquacadda_Nuxis_2019_def/D_posizionamento saggi_NUXIS 2019","Acquacadda_Nuxis_2019_def/E_piante fine scavo vettoriali_NUXIS 2019" -"Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)","Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)/GNA_SAN BASILIO_2019/GNA_topografia_San Basilio/POSIZIONAMENTO SAGGI E AREE DELLE INDAGINI","Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)/GNA_SAN BASILIO_2019/GNA_topografia_San Basilio/PLANIMETRIE SAGGI AREE DELLE INDAGINI" -"Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)","Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)/S.Basilio-UniVe_GNA/D.GNA_Posizionamento_San Basilio","Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)/S.Basilio-UniVe_GNA/E-F.GNA_Pianta fine scavo_San Basilio/GNA_Pianta di fine scavo_San Basilio" -"C_F_GNA_Oscurusciuto_2020","C_F_GNA_Oscurusciuto_2020/D-E-F_GNA_topografia_Oscurusciuto/Posizionamento_limiti_saggi_indagati_Oscurusciuto_vettoriale","C_F_GNA_Oscurusciuto_2020/D-E-F_GNA_topografia_Oscurusciuto/Pianta_fine_scavo_Oscurusciuto_2019_vettoriale" -"Cerchiara-Damale_rev","Cerchiara-Damale_rev/D. Posizionamento saggio","Cerchiara-Damale_rev/D. Posizionamento saggio/E. Pianta fine scavo_vettoriale" -"Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)","Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)/IVa_E_Tav.1_Shape Posizionamento area scavo","Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)/IVa_E_Tav.1_Shape Posizionamento area scavo" -"Cervia Vecchia_rev","Cervia Vecchia_rev/D_GNA_topografia_Cervia","Cervia Vecchia_rev/D_GNA_topografia_Cervia" -"Civitavecchia (RM)_loc. Ficoncella_Aquae Tauri_docum. IVa_2019","Civitavecchia (RM)_loc. Ficoncella_Aquae Tauri_docum. IVa_2019/Civitavecchia (RM)_Ficoncella_documentazione fine scavo 2019_IVa/IVa_D_E_F_topografia_Aquae_Tauri/Posizionamento saggi 2019","Civitavecchia (RM)_loc. Ficoncella_Aquae Tauri_docum. IVa_2019/Civitavecchia (RM)_Ficoncella_documentazione fine scavo 2019_IVa/IVa_D_E_F_topografia_Aquae_Tauri/Pianta di fine scavo 2019" -"Frascineto Timpone delle Fave_rev","Frascineto Timpone delle Fave_rev/D. Posizionamento dell'area","Frascineto Timpone delle Fave_rev/D. Posizionamento dell'area" -"GNA_AQUILEIAcomellimoro_2019_rev","GNA_AQUILEIAcomellimoro_2019_rev/GNA_topografia_AQUILEIAcomellimoro_2019/D.Posizionamento_AQU19COM","GNA_AQUILEIAcomellimoro_2019_rev/GNA_topografia_AQUILEIAcomellimoro_2019/E.Planimetria generale_AQU19COM" -"GNA_Poggio Pimperiale_Poggibonsi","GNA_Poggio Pimperiale_Poggibonsi/5591449/GNA_topografia_PoggioImperiale/POSIZIONAMENTO","GNA_Poggio Pimperiale_Poggibonsi/5591449/GNA_topografia_PoggioImperiale/PLANIMETRIE SAGGI/SHAPE A22" -"Jesolo (Ve) - Loc. San Mauro e Torre del Caligo","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/GNA_topografia_Jesolo/GIS_shp","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/GNA_topografia_Jesolo/GIS_shp" -"Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)","Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)/IVa_D_E_F_GNA_topografia_Via Nomentum-Eretum","Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)/IVa_D_E_F_GNA_topografia_Via Nomentum-Eretum" -"Quarto d'Altino -Loc. Fornace","Quarto d'Altino -Loc. Fornace/Cupitò/GNA_Altino_posizionamento_shp","Quarto d'Altino -Loc. Fornace/Cupitò/GNA_Altino_posizionamento_shp" -"Tolfa (RM)_Bufalareccia_documentazione IVa 2019","Tolfa (RM)_Bufalareccia_documentazione IVa 2019/IVa_D_GNA_posizionamento dei limiti in formato vectoriale_Bufalareccia 2019","Tolfa (RM)_Bufalareccia_documentazione IVa 2019/IVa_E_GNA_pianta di fine scavo multipolygon_Bufalareccia 2019" -"UNIME_ LAINO_REV","UNIME_ LAINO_REV/D. Limiti saggi/laino 2019 limiti saggi","UNIME_ LAINO_REV/D. Limiti saggi/laino 2019 limiti saggi" -"UNIME_TORTORA_rev","UNIME_TORTORA_rev/tortora 2019 limiti saggi","UNIME_TORTORA_rev/tortora 2019 pianta fine scavo" -"UsiniTomestighes","UsiniTomestighes/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/D_GNA_Posizionamento_limiti_aree_indagate","UsiniTomestighes/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/E_GNA_Piante_fine_scavo_Tomestighes/Area A_est" \ No newline at end of file diff --git a/test-data/concessioni/immagine.png b/test-data/concessioni/immagine1.png similarity index 100% rename from test-data/concessioni/immagine.png rename to test-data/concessioni/immagine1.png diff --git a/test-data/concessioni/DATASET_GNA_01.csv b/test-data/concessioni/packages/DATASET_GNA_01.csv similarity index 100% rename from test-data/concessioni/DATASET_GNA_01.csv rename to test-data/concessioni/packages/DATASET_GNA_01.csv diff --git a/test-data/concessioni/DATASET_GNA_02.csv b/test-data/concessioni/packages/DATASET_GNA_02.csv similarity index 100% rename from test-data/concessioni/DATASET_GNA_02.csv rename to test-data/concessioni/packages/DATASET_GNA_02.csv diff --git a/test-data/concessioni/packages/concessioni04-03.csv b/test-data/concessioni/packages/concessioni04-03.csv new file mode 100644 index 0000000..2294fbc --- /dev/null +++ b/test-data/concessioni/packages/concessioni04-03.csv @@ -0,0 +1,17 @@ +"Acquacadda_Nuxis_2019_def","Acquacadda_Nuxis_2019_def/D_posizionamento saggi_NUXIS 2019/Areale saggi Nuxis - Grotta di Acquacadda.shp" +"Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)","Ariano nel Polisine (Ro) - Loc. San Basilio ( Unvi. di Padova)/GNA_SAN BASILIO_2019/GNA_topografia_San Basilio/POSIZIONAMENTO SAGGI E AREE DELLE INDAGINI/San Basilio 2019_EPSG32633.shp" +"Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)","Ariano Polesine (Ro) - Loc. San Basilio (Uni. di Venezia)/S.Basilio-UniVe_GNA/D.GNA_Posizionamento_San Basilio/limiti di scavo.shp" +"C_F_GNA_Oscurusciuto_2020","C_F_GNA_Oscurusciuto_2020/D-E-F_GNA_topografia_Oscurusciuto/Posizionamento_limiti_saggi_indagati_Oscurusciuto_vettoriale/Oscurusciuto_area_scavo_2019.shp" +"Cerchiara-Damale_rev","Cerchiara-Damale_rev/D. Posizionamento saggio/RB228_trench2.shp" +"Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)","Cerveteri (RM)_loc. Monte Abatone_documentazione fine scavo 2019 (paragr. IVa)/IVa_E_Tav.1_Shape Posizionamento area scavo/Posizionamento Area di scavo Monte Abatone 2019.shp" +"Cervia Vecchia_rev","Cervia Vecchia_rev/D_GNA_topografia_Cervia/CE19_09.shp" +"Frascineto Timpone delle Fave_rev","Frascineto Timpone delle Fave_rev/D. Posizionamento dell'area/TDF2019_ricognizione.shp" +"GNA_AQUILEIAcomellimoro_2019_rev","GNA_AQUILEIAcomellimoro_2019_rev/GNA_topografia_AQUILEIAcomellimoro_2019/D.Posizionamento_AQU19COM/Tav01_line_posizionamento_EPSG4326.shp" +"GNA_Poggio Pimperiale_Poggibonsi","GNA_Poggio Pimperiale_Poggibonsi/5591449/GNA_topografia_PoggioImperiale/POSIZIONAMENTO/POSIZIONAMENTO A22.shp" +"Jesolo (Ve) - Loc. San Mauro e Torre del Caligo","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/GNA_topografia_Jesolo/GIS_shp/Area di Scavo 2019.shp" +"Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)","Monterotondo (RM)_loc. Tor Mancina_documentazione fine scavo 2019 (paragr. IVa)/IVa_D_E_F_GNA_topografia_Via Nomentum-Eretum/pianta fine scavo fasi.dxf" +"Quarto d'Altino -Loc. Fornace","Quarto d'Altino -Loc. Fornace/Cupitò/GNA_Altino_posizionamento_shp/ALT19_area survey_EPSG4326.shp" +"Tolfa (RM)_Bufalareccia_documentazione IVa 2019","Tolfa (RM)_Bufalareccia_documentazione IVa 2019/IVa_D_GNA_posizionamento dei limiti in formato vectoriale_Bufalareccia 2019/BUFFA_POLYGON_.shp" +"UNIME_ LAINO_REV","UNIME_ LAINO_REV/D. Limiti saggi/laino 2019 limiti saggi/SAGGI.shp" +"UNIME_TORTORA_rev","UNIME_TORTORA_rev/tortora 2019 limiti saggi/SAGGI 2019.shp" +"UsiniTomestighes","UsiniTomestighes/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/D_GNA_Posizionamento_limiti_aree_indagate/Limiti.shp" \ No newline at end of file diff --git a/test-data/concessioni/packages/concessioni04-03_filtered.csv b/test-data/concessioni/packages/concessioni04-03_filtered.csv new file mode 100644 index 0000000..d8ae97d --- /dev/null +++ b/test-data/concessioni/packages/concessioni04-03_filtered.csv @@ -0,0 +1 @@ +"UsiniTomestighes","UsiniTomestighes/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/D_GNA_Posizionamento_limiti_aree_indagate/Limiti.shp" \ No newline at end of file diff --git a/test-data/concessioni/concessioni_23_04.csv b/test-data/concessioni/packages/concessioni_23_04.csv similarity index 100% rename from test-data/concessioni/concessioni_23_04.csv rename to test-data/concessioni/packages/concessioni_23_04.csv diff --git a/test-data/concessioni/invio_08_05.csv b/test-data/concessioni/packages/invio_08_05.csv similarity index 100% rename from test-data/concessioni/invio_08_05.csv rename to test-data/concessioni/packages/invio_08_05.csv diff --git a/test-data/concessioni/transferTest.json b/test-data/concessioni/transferTest.json new file mode 100644 index 0000000..4f0a3f4 --- /dev/null +++ b/test-data/concessioni/transferTest.json @@ -0,0 +1 @@ +{"mongo_id":"6157089e596924b380a174b7","id":0,"recordType":"CONCESSIONE","version":"1.0.0","licenzaID":"CC-BY","policy":"OPEN","nome":"Concessione : publish test ","folderId":"d59b3580-4aeb-4ec0-81a2-0aee233c8547","lastUpdateTime":"2021-10-01T15:09:50.468","lastUpdateUser":null,"creationTime":"2021-10-01T15:09:50.468","creationUser":null,"report":null,"introduzione":"This is my MONGO project","descrizioneContenuto":"It contains this and that","authors":["Some one","Some, oneelse"],"contributore":"Contrib 1","titolari":["Some one","Some, oneelse"],"responsabile":"Someone","editore":"Editore","fontiFinanziamento":["Big pharma","Pentagon"],"soggetto":["Research Excavation","Archeology"],"risorseCorrelate":null,"dataInizioProgetto":"2021-10-01T15:09:30.775","dataFineProgetto":"2021-10-01T15:09:30.778","titolareLicenza":["Qualcun altro"],"titolareCopyright":["Chiedilo in giro"],"paroleChiaveLibere":["Robba","Stuff"],"paroleChiaveICCD":["vattelapesca","somthing something"],"centroidLat":null,"centroidLong":null,"relazioneScavo":{"mongo_id":"615708b7596924b380a174bc","id":0,"policy":"OPEN","licenseID":"CC-BY","titolo":"Concessione : publish test relazione di scavo","creationTime":"2021-10-01T15:09:50.468","actualContent":[{"type":"WorkspaceContent","id":0,"mimetype":"application/pdf","storageID":"fd244602-ad16-4acb-a7c0-e7172d5e9572","link":"https://data.dev.d4science.org/shub/E_aXI3OFRxQXY3d1VHZ21JQ3p1ajJuS2NOVEZNV1pYYmg2N1hSM09yVFQxSDRDK3BWZ2VPM0hqVk5sT1ZzYllXYw==","name":"relazione.pdf"}],"responsabili":["Some one","Some, oneelse"],"soggetto":["Research Excavation","Archeology"]},"abstractRelazione":{"mongo_id":"615708c6596924b380a174c0","id":0,"policy":"OPEN","licenseID":"CC-BY","titolo":"Concessione : publish test abstract relazione di scavo","creationTime":"2021-10-01T15:09:50.468","actualContent":[{"type":"WorkspaceContent","id":0,"mimetype":"application/pdf","storageID":"9c89d891-f96e-4b5d-9498-84dc2351205b","link":"https://data.dev.d4science.org/shub/E_WTVyc2xyZlZqREZiM0s2WXc1ZVJyc1F4T2tsYlF5TEpEYmRnbVRQaHZnL0h0ODNSeG9KMExtV0pzcXBrd1Niaw==","name":"relazione.pdf"}],"abstractIta":"Il mio abstract","abstractEng":"My abstract"},"immaginiRappresentative":[{"mongo_id":"615709a0596924b380a174f6","id":0,"policy":"OPEN","licenseID":"CC-BY-4.0","titolo":"My image number 0","creationTime":"2021-10-01T15:09:30.779","actualContent":[{"type":"WorkspaceContent","id":0,"mimetype":"image/png","storageID":"75f1d578-dd22-4c3b-bab3-75ba39e2b085","link":"https://data.dev.d4science.org/shub/E_a20xMTlROWlYRllQMXVDcjlxenpaaVJwT1VJU1lhVzk4bnVHNWhOUjVTWUNlblVUNnlZcDJ5M1NLdVdkb0VPZw==","name":"immagine.png"}],"didascalia":"You can see my image number 0","format":"TIFF","responsabili":["Some one","Some, oneelse"],"soggetto":["Research Excavation","Archeology"]}],"posizionamentoScavo":{"mongo_id":"6157092d596924b380a174d9","id":0,"policy":"OPEN","licenseID":"CC-BY-4.0","titolo":"Concessione : publish test posizionamento scavo","creationTime":"2021-10-01T15:09:50.468","actualContent":[{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"1e366a00-3060-4e12-b14f-fa909cbbbd3b","link":"https://data.dev.d4science.org/shub/E_RFR1bU13dEtKc1Q2TXZ6ZFBmYXFsbmEzUDF5Yi9zWXpoNURiUlJITGVaS2pGL0QwWGJwemFUY25BS0xIQWFobA==","name":"pianta.sbx"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"8cc2876a-f98f-4a1b-9f64-b581323324c3","link":"https://data.dev.d4science.org/shub/E_OVZKNmtpanJnYnBLRUcrTGJQMXdUSGc0OGFBd2d0Y1JkNUwvWVBUdjlHdDRUSnlQYlB6UW5mMzE2cEs4bGkweg==","name":"pianta.shx"},{"type":"WorkspaceContent","id":0,"mimetype":"text/plain","storageID":"2f2e04ff-9a80-4f83-a25f-cffa2bc6452c","link":"https://data.dev.d4science.org/shub/E_M2k3cVlSbE5OYmZ6Ukp1Sm54ZlNIUyt5b2tWTUk0amIvMjFlWjJsTkw3YjBSSzQvUklvTWRCZUgyV2FoUjhUaA==","name":"pianta.cpg"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"e2a691c0-a5aa-43ef-bd8c-ce27064546b8","link":"https://data.dev.d4science.org/shub/E_eUx4R1RCSDgrdG5EWTV5MlFYSm9QWmxOSWdlcDR6TzNrc3BKVjhnaDhkWFQ1MzM4bjlldHhRa0FRYk5QL1NrMg==","name":"pianta.shp"},{"type":"WorkspaceContent","id":0,"mimetype":"application/xml","storageID":"9f4cccc7-81e5-4815-a6fc-802f19f8b511","link":"https://data.dev.d4science.org/shub/E_TkNRbGhFby9jWWs1a2Vac0lkei9hTmg3VWJ5Z1VUZmpPUUJJY2VHV2FLeE1OSm1JMDVqeGRKK0E0dll3cXFVNA==","name":"pianta.shp.xml"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-dbf","storageID":"8e639a94-322e-4a5d-a39f-f34262bdebec","link":"https://data.dev.d4science.org/shub/E_Smx5eVFyRUZSYldNVHJQMXFzdFdpd3pqNm80aS82YVJuMWJUMGw4T05PMnQvNG84OEwzL2dFejk0QTlsSGJPYQ==","name":"pianta.dbf"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"cb66cf81-b357-4fdc-9f37-a51d7d4f368e","link":"https://data.dev.d4science.org/shub/E_dXZnaTZoVU92enpxVTIyYWhIaUhwQWV1RDE2L2JaWVZ2Z0pidmx2V25oMDFsR01CVlkzMmRqOE83UDZKN2lKSw==","name":"pianta.sbn"},{"type":"WorkspaceContent","id":0,"mimetype":"text/plain","storageID":"d0f7f683-7b67-44ba-9618-b9321c783657","link":"https://data.dev.d4science.org/shub/E_TFAxNEM1VEJIUlV6eDN1K2FMTXdYZHBCVE9vTnNGaHIvL2s5czJlK3BCbVdXcDJQTjFOQ0JjeHN1V2h2d1J1eA==","name":"pianta.prj"}],"layerUUID":null,"layerID":null,"layerName":null,"wmsLink":null,"workspace":null,"abstractSection":"Posizionamento topografico georeferenziato dell’area interessata dalle indagini","topicCategory":"Society","subTopic":"Archeology","bbox":{"maxLat":90.0,"maxLong":180.0,"minLat":-90.0,"minLong":-180.0},"paroleChiaveLibere":["Robba","Stuff"],"paroleChiaveICCD":["vattelapesca","somthing something"],"valutazioneQualita":"Secondo me si","metodoRaccoltaDati":"Fattobbene","scalaAcquisizione":"1:10000","authors":["Some one","Some, oneelse"],"responsabile":"Someone"},"pianteFineScavo":[{"mongo_id":"6157098d596924b380a174f2","id":0,"policy":"RESTRICTED","licenseID":"CC-BY-4.0","titolo":"Pianta from /GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/E_GNA_Piante_fine_scavo_Tomestighes/Area A_est/area di concessione.shp","creationTime":"2021-10-01T15:09:50.468","actualContent":[{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"52f64ba0-7e93-440f-9cd1-4b8286e96e2f","link":"https://data.dev.d4science.org/shub/E_dzBDTW95ZXlSM0I5NjkyOThRZXEvVk9nN2Yya0NkOTdpMGt2alNZTjh6VDVRYTcxcUpMZU94a3lOTitrR1hIcQ==","name":"pianta.sbx"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"c5031168-f3f2-4a9f-93e5-f71a0b3a9cb5","link":"https://data.dev.d4science.org/shub/E_YitWN2NlOXd6NXV4aVIvcW00ZWNlTGtSeHoyNVF4ZnhNZXh1dTZvdlkzTTdwdm9GbGJ0OVg2ZzFNdGo0dGZ0WA==","name":"pianta.shx"},{"type":"WorkspaceContent","id":0,"mimetype":"text/plain","storageID":"7aaabd97-01df-4b55-bee6-5bd1ab628d02","link":"https://data.dev.d4science.org/shub/E_WDljaFdNSHZIaTlhMjcxdUZRR0EwYUNPVzJ6Ry90VUVlOGtzSnc0c04wZXlaNzAySENBcUc1Yi9nekVqRjdMMQ==","name":"pianta.cpg"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"7e577212-d572-44da-b840-3d1da2746e09","link":"https://data.dev.d4science.org/shub/E_MGZuVyt3c0RFWTVTTktvbVdnVEhZcmgyMkNXdExtTUMvTmUxOVc4cDVwNFd3SW1xTndiSUNQUzkxajFlMEdJTw==","name":"pianta.shp"},{"type":"WorkspaceContent","id":0,"mimetype":"application/xml","storageID":"e2839deb-2d21-4193-b1c4-3cf1db0aa529","link":"https://data.dev.d4science.org/shub/E_empENTJPWFNZUC90aEdZR2tmR3U0aHpuUDNTOWd4RVo0SW9KVUc0c0prcDVjb3VpSitESG1CdmtGQlJxT2VYNQ==","name":"pianta.shp.xml"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-dbf","storageID":"ac8aa6ff-7bbf-4d15-bf0a-c9012374152a","link":"https://data.dev.d4science.org/shub/E_T2hJOTVLbU9ncGN1TXhRanFEVytodDNvRUpJOHJOTXhrOUFJRFJVbzl4ZWRGREFSQlZjOFNjYmFjUGlQK0Q3Qw==","name":"pianta.dbf"},{"type":"WorkspaceContent","id":0,"mimetype":"application/x-shapefile","storageID":"6ec0d074-3bd4-430a-bceb-f056bec00690","link":"https://data.dev.d4science.org/shub/E_RjJNTUZIM1ZtZk5qUXcyRE44Sk51K0c3dHpxSWJWeG5iZUNrQklDMzgyT1lXcjY3U3ZNTk5WZ2swOWhZRHg2dQ==","name":"pianta.sbn"},{"type":"WorkspaceContent","id":0,"mimetype":"text/plain","storageID":"a0b71dc9-91d8-4b7f-8167-c5fd1f76f075","link":"https://data.dev.d4science.org/shub/E_VkRlSkZvQ0p4Z3pkNTN1cVdjVUpDeHpqTmZXOXhQL1VxbHhNeGlEWlNmb21rMy9qTlg2UGVQeUp1STd2TjFPUA==","name":"pianta.prj"}],"layerUUID":null,"layerID":null,"layerName":null,"wmsLink":null,"workspace":null,"abstractSection":"Planimetria georeferenziata dell'area indagata al termine delle attività","topicCategory":"Society","subTopic":"Archeology","bbox":{"maxLat":90.0,"maxLong":180.0,"minLat":-90.0,"minLong":-180.0},"paroleChiaveLibere":["Robba","Stuff"],"paroleChiaveICCD":["vattelapesca","somthing something"],"valutazioneQualita":"Secondo me si","metodoRaccoltaDati":"Fattobbene","scalaAcquisizione":"1:10000","authors":["Some one","Some, oneelse"],"responsabile":"Someone"}],"genericContent":[]} \ No newline at end of file diff --git a/use-cases/pom.xml b/use-cases/pom.xml index be6be9f..73ba932 100644 --- a/use-cases/pom.xml +++ b/use-cases/pom.xml @@ -23,6 +23,27 @@ ${gitBaseUrl}/${project.artifactId}.git + + + + + + org.gcube.distribution + gcube-bom + 2.0.1 + pom + import + + + org.gcube.distribution + gcube-smartgears-bom + 2.1.0 + pom + import + + + + org.gcube.application diff --git a/use-cases/src/main/java/org/gcube/application/cms/usecases/ClearConcessioni.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/ClearConcessioni.java similarity index 95% rename from use-cases/src/main/java/org/gcube/application/cms/usecases/ClearConcessioni.java rename to use-cases/src/test/java/org/gcube/application/cms/usecases/ClearConcessioni.java index e3a6b4d..40161a5 100644 --- a/use-cases/src/main/java/org/gcube/application/cms/usecases/ClearConcessioni.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/ClearConcessioni.java @@ -29,8 +29,8 @@ public class ClearConcessioni { Iterator it=null; - it=manager.getList(); -// it=manager.search("{\"centroidLat\" : 0}"); +// it=manager.getList(); + it=manager.search("{\"nome\" : {\"$regex\" : \"Mock.*\"}}"); it.forEachRemaining((Concessione c)->{ try{ diff --git a/use-cases/src/main/java/org/gcube/application/cms/usecases/EditFileSet.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/EditFileSet.java similarity index 97% rename from use-cases/src/main/java/org/gcube/application/cms/usecases/EditFileSet.java rename to use-cases/src/test/java/org/gcube/application/cms/usecases/EditFileSet.java index 126c4c2..2bffb60 100644 --- a/use-cases/src/main/java/org/gcube/application/cms/usecases/EditFileSet.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/EditFileSet.java @@ -18,7 +18,7 @@ public class EditFileSet { public static void main(String[] args) throws Exception { // params - String context="/gcube/devsec/devVRE"; + String context= "/org/gcube/devsec/devVRE"; String publishOption="true"; String toUpdateId="6131f42502ad3d2580412da7"; diff --git a/use-cases/src/main/java/org/gcube/application/cms/usecases/Export.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/Export.java similarity index 99% rename from use-cases/src/main/java/org/gcube/application/cms/usecases/Export.java rename to use-cases/src/test/java/org/gcube/application/cms/usecases/Export.java index 4c7f8b6..22c596b 100644 --- a/use-cases/src/main/java/org/gcube/application/cms/usecases/Export.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/Export.java @@ -35,7 +35,7 @@ public class Export { File dir= new File("/Users/fabioisti/git/geoportal-client/import1628178107083"); // String targetContext="/pred4s/preprod/preVRE"; - String targetContext="/gcube/devsec/devVRE"; + String targetContext= "/org/gcube/devsec/devVRE"; diff --git a/use-cases/src/main/java/org/gcube/application/cms/usecases/Import.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/Import.java similarity index 100% rename from use-cases/src/main/java/org/gcube/application/cms/usecases/Import.java rename to use-cases/src/test/java/org/gcube/application/cms/usecases/Import.java diff --git a/use-cases/src/main/java/org/gcube/application/cms/usecases/RepublishAll.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/RepublishAll.java similarity index 89% rename from use-cases/src/main/java/org/gcube/application/cms/usecases/RepublishAll.java rename to use-cases/src/test/java/org/gcube/application/cms/usecases/RepublishAll.java index 15301c9..1577f71 100644 --- a/use-cases/src/main/java/org/gcube/application/cms/usecases/RepublishAll.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/RepublishAll.java @@ -6,6 +6,7 @@ import org.gcube.application.geoportal.client.legacy.ConcessioniManagerI; import org.gcube.application.geoportal.common.model.legacy.Concessione; import java.util.ArrayList; +import java.util.Iterator; import java.util.concurrent.atomic.AtomicLong; import static org.gcube.application.geoportal.client.GeoportalAbstractPlugin.statefulMongoConcessioni; @@ -24,7 +25,11 @@ public class RepublishAll { AtomicLong count=new AtomicLong(0); AtomicLong nullCount=new AtomicLong(0); AtomicLong errCount=new AtomicLong(0); - manager.getList().forEachRemaining((Concessione c)->{ + Iterator it=null; +// it=manager.getList(); + it=manager.search("{\"report.status\" : {$eq : \"WARNING\"}}"); + + it.forEachRemaining((Concessione c)->{ try{ String currentId=c.getMongo_id(); if(currentId==null) { diff --git a/use-cases/src/main/java/org/gcube/application/cms/usecases/RepublishSingle.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/RepublishSingle.java similarity index 86% rename from use-cases/src/main/java/org/gcube/application/cms/usecases/RepublishSingle.java rename to use-cases/src/test/java/org/gcube/application/cms/usecases/RepublishSingle.java index 3789e00..ad9d37e 100644 --- a/use-cases/src/main/java/org/gcube/application/cms/usecases/RepublishSingle.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/RepublishSingle.java @@ -1,7 +1,6 @@ package org.gcube.application.cms.usecases; import org.gcube.application.cms.tests.TokenSetter; -import org.gcube.application.geoportal.client.legacy.ConcessioniManagerI; import org.gcube.application.geoportal.common.model.legacy.Concessione; import org.gcube.application.geoportal.common.rest.MongoConcessioni; @@ -13,7 +12,7 @@ public class RepublishSingle { public static void main(String[] args) throws Exception { TokenSetter.set("/gcube/devsec/devVRE"); - String id="6138c3a002ad3d1f0cd659f4"; + String id="6137497102ad3d1f0cd6586b"; MongoConcessioni manager=mongoConcessioni().build(); diff --git a/use-cases/src/main/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java similarity index 62% rename from use-cases/src/main/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java rename to use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java index 85a7240..e25f6d4 100644 --- a/use-cases/src/main/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java @@ -4,11 +4,13 @@ import com.opencsv.CSVReader; import lombok.extern.slf4j.Slf4j; import org.gcube.application.cms.tests.model.TestModel; import org.gcube.application.cms.tests.TokenSetter; +import org.gcube.application.geoportal.client.utils.Serialization; import org.gcube.application.geoportal.common.utils.FileSets; import org.gcube.application.geoportal.common.model.legacy.Concessione; import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport; import org.gcube.application.geoportal.common.rest.MongoConcessioni; +import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.StorageUtils; import java.io.*; @@ -22,28 +24,29 @@ public class MockFromFolder { public static void main(String[] args) throws Exception { //PARAMS - String context="/gcube/devsec/devVRE"; + String context= "/gcube/devsec/devVRE"; + File descriptorsBaseFolder=new File(TestModel.getBaseFolder(),"packages"); //Concessioni 04-03 - //String packageBaseDir="/Users/fabioisti/Documents/Concessioni 04-03/"; - //String csvDescriptor="src/test/resources/concessioni/concessioni04-03.csv"; + String packageBaseDir="/Users/fabioisti/Documents/Concessioni 04-03/"; + String csvDescriptor="concessioni04-03_filtered.csv"; //DATASET_GNA_01 // String packageBaseDir="/Users/fabioisti/Documents/DATASET_GNA_01"; -// String csvDescriptor="src/test/resources/concessioni/DATASET_GNA_01.csv"; +// String csvDescriptor="DATASET_GNA_01.csv"; //DATASET_GNA_02 //String packageBaseDir="/Users/fabioisti/Documents/DATASET_GNA_02"; - //String csvDescriptor="src/test/resources/concessioni/DATASET_GNA_02.csv"; + //String csvDescriptor="DATASET_GNA_02.csv"; // invio_08_02 // String packageBaseDir="/Users/fabioisti/Documents/invio_08_05"; -// String csvDescriptor="src/test/resources/concessioni/invio_08_05.csv"; +// String csvDescriptor="invio_08_05.csv"; // concessioni 23_04 - String packageBaseDir="/Users/fabioisti/Documents/Concessioni_23_04"; - String csvDescriptor="src/test/resources/concessioni/concessioni_23_04.csv"; +// String packageBaseDir="/Users/fabioisti/Documents/Concessioni_23_04"; +// String csvDescriptor="concessioni_23_04.csv"; @@ -62,7 +65,7 @@ public class MockFromFolder { File baseDir=new File(packageBaseDir); ArrayList pushed=new ArrayList<>(); - CSVReader reader = new CSVReader(new FileReader(csvDescriptor)); + CSVReader reader = new CSVReader(new FileReader(new File(descriptorsBaseFolder,csvDescriptor))); String [] nextLine; //reads one line at a time while ((nextLine = reader.readNext()) != null) @@ -71,26 +74,30 @@ public class MockFromFolder { //Create new String projectName = nextLine[0]; String positionPath = nextLine[1]; - String piantePath = nextLine[2]; try { //NB raggruppa per file - Map.Entry> posSets = clusterizeFiles(positionPath, baseDir).entrySet().stream().findFirst().get(); - Map> pianteSets = clusterizeFiles(piantePath, baseDir); + Map.Entry> posSets = Files.getAllShapeSet(new File(baseDir,positionPath),true). + entrySet().stream().findFirst().get(); + Map> pianteSets = Files.getAllShapeSet(new File(baseDir,projectName),true); - // Sometimes they are the same - if(positionPath.equals(piantePath)) + if(pianteSets.size()>1) pianteSets.remove(posSets.getKey()); + + log.debug("Entry {} pos Size {} piante {} ",projectName,posSets.getValue().size(),pianteSets.size()); - Concessione c = createMock(projectName, pianteSets, posSets.getValue(), client, storage); + + Concessione c = createMock(projectName,baseDir.getName(), pianteSets, posSets.getValue(), client, storage); publishedCount++; if (c.getReport().getStatus().equals(ValidationReport.ValidationStatus.PASSED)) successcount++; pushed.add(c); + + }catch(Throwable t){ System.err.println("Problematic entry "+projectName); t.printStackTrace(System.err); @@ -111,46 +118,40 @@ public class MockFromFolder { - private static Map> clusterizeFiles(String basePath,File packageFolder) throws IOException { - log.debug("Clusterizing "+basePath); - HashMap> toReturn = new HashMap<>(); - File baseDir=new File(packageFolder,basePath); - for(File shp:baseDir.listFiles((dir,name)->{return name.endsWith(".shp");})) { - String basename=shp.getName().substring(0,shp.getName().lastIndexOf(".")); - List fileset=new ArrayList<>(); - for (File shpSet : baseDir.listFiles((dir, name) -> {return name.startsWith(basename);})) - fileset.add(shpSet); - log.debug("SHP {} Set size {} ",basename,fileset.size()); - toReturn.put(basename,fileset); - } - return toReturn; - } - private static Concessione createMock(String baseName,Map> piante, List pos, + private static Concessione createMock(String baseName,String packageName,Map> piante, List pos, MongoConcessioni client, StorageUtils storage) throws Exception { Concessione c= TestModel.prepareConcessione(piante.size(), 2); - c.setNome("Mock for "+baseName); + c.setNome("Mock for "+baseName+" ("+packageName+")"); c= client.createNew(c); String mongoId=c.getMongo_id(); // TEST DATA, DO NOT CARE client.registerFileSet(mongoId, FileSets.prepareRequest(storage, - Concessione.Paths.RELAZIONE,new File ("src/test/resources/concessioni/relazione.pdf"))); + Concessione.Paths.RELAZIONE,new File (TestModel.getBaseFolder(),"relazione.pdf"))); client.registerFileSet(mongoId, FileSets.prepareRequest(storage, - Concessione.Paths.imgByIndex(0),new File("src/test/resources/concessioni/immagine.png"))); + Concessione.Paths.imgByIndex(0),new File(TestModel.getBaseFolder(),"immagine.png"))); // POSIZIONAMENTO client.registerFileSet(mongoId, FileSets.prepareRequest(storage, Concessione.Paths.POSIZIONAMENTO,pos.toArray(new File[pos.size()]))); + // PIANTE Map.Entry>[] entries= piante.entrySet().toArray(new Map.Entry[0]); - for( int i= 0; i< piante.size();i++) - client.registerFileSet(mongoId, FileSets.prepareRequest(storage, - Concessione.Paths.piantaByIndex(i),entries[i].getValue().toArray(new File[0]))); + for( int i= 0; i< piante.size();i++) { + // Set layer name + c=client.getById(mongoId); + String path=Concessione.Paths.piantaByIndex(i); + c.getContentByPath(path).setTitolo(" Pianta from "+entries[i].getKey()); + client.update(mongoId, Serialization.write(c)); + + //Set fileset + client.registerFileSet(mongoId, FileSets.prepareRequest(storage,path, entries[i].getValue().toArray(new File[0]))); + } c=client.publish(mongoId);