diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java index 45930a6..1bb66bd 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java @@ -21,7 +21,7 @@ public class Files { Map shpAbsolutePath -> fileset */ - private static Map> clusterizeFilesByExtension(File base,String extension,Boolean recursive) throws IOException { + public static Map> clusterizeFilesByExtension(File base,String extension,Boolean recursive) throws IOException { HashMap> toReturn = new HashMap<>(); log.debug("Clustering "+base.getAbsolutePath()); List targetFiles=new ArrayList<>(); @@ -48,7 +48,7 @@ public class Files { } - private static List getSiblings(File location,String baseName){ + public static List getSiblings(File location,String baseName){ List fileset=new ArrayList<>(); for (File shpSet : location.listFiles((dir, name) -> {return name.startsWith(baseName);})) fileset.add(shpSet); diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/ImplementationProvider.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/ImplementationProvider.java index b1c069b..ae629f6 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/ImplementationProvider.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/ImplementationProvider.java @@ -51,8 +51,8 @@ public class ImplementationProvider { public void shutdown() { // Stop JPA // AbstractRecordManager.shutdown(); - mongoConnectionProvider.shustdown(); - mongoClientProvider.shustdown(); + mongoConnectionProvider.shutdown(); + mongoClientProvider.shutdown(); } public void startup() { diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/AbstractScopedMap.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/AbstractScopedMap.java index 50f4871..4d4cdae 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/AbstractScopedMap.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/AbstractScopedMap.java @@ -39,18 +39,21 @@ public abstract class AbstractScopedMap implements Engine{ } if(TTL!=null) { - if(!found.getCreationTime().plus(TTL).isBefore(LocalDateTime.now())) { + if(found.getCreationTime().plus(TTL).isBefore(LocalDateTime.now())) { log.debug(name+" : elapsed TTL, disposing.."); dispose(found.getTheObject()); - found=scopeMap.put(currentScope, new TTLObject(LocalDateTime.now(),retrieveObject())); + TTLObject newer=new TTLObject(LocalDateTime.now(),retrieveObject()); + scopeMap.put(currentScope, newer); + found=scopeMap.get(currentScope); } }else {log.debug(name+" : TTL is null, never disposing..");} + log.debug("Returning {} ",found); return found.getTheObject(); } @Override - public void shustdown() { + public void shutdown() { log.warn(name + ": shutting down"); scopeMap.forEach((String s,TTLObject o)->{ try{if(o!=null&&o.getTheObject()!=null) diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/Engine.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/Engine.java index 0e93075..0f17b8d 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/Engine.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/Engine.java @@ -6,7 +6,7 @@ import org.gcube.application.geoportal.service.model.internal.faults.Configurati public interface Engine { public void init(); - public void shustdown(); + public void shutdown(); public T getObject() throws ConfigurationException; } diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java index 7709cd1..48e6fdb 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java @@ -37,8 +37,7 @@ public class StorageClientProvider extends AbstractScopedMap { @Override protected void dispose(StorageUtils toDispose) { try { - //TODO ASK -// toDispose.close(); + //TODO }catch (NullPointerException e) { // expected if closed without uploading }catch(Throwable t) { diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageHubProvider.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageHubProvider.java index dc549bc..29aaef1 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageHubProvider.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageHubProvider.java @@ -17,7 +17,7 @@ public class StorageHubProvider implements Engine { } @Override - public void shustdown() { + public void shutdown() { // TODO Auto-generated method stub } diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java index 7b76bff..b8ffc04 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ConcessioniOverMongoTest.java @@ -310,11 +310,10 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ private Concessione getFullPublished(WebTarget target) throws Exception { - File layerFolder=new File( - "/Users/fabioisti/Documents/Concessioni 04-03/UsiniTomestighes"); + File layerFolder=TestModel.getBaseFolder(); Map> layers = Files.getAllShapeSet(layerFolder,true); - Concessione c=TestModel.prepareConcessione(1,1); + Concessione c=TestModel.prepareConcessione(layers.size(),1); c.setNome("Concessione : publish test "); StorageUtils storage=new StorageUtils(); @@ -330,8 +329,8 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ String[] keys=layers.keySet().toArray(new String [0]); c=upload(storage,target,c.getMongo_id(),Paths.POSIZIONAMENTO, - TestModel.getBaseFolder().list((file,name)->{return name.startsWith("pianta.shp");})); -// layers.get(keys[0]).toArray(new File[0])); +// TestModel.getBaseFolder().list((file,name)->{return name.startsWith("pianta.shp");})); + layers.get(keys[0]).toArray(new File[0])); // Clash on workspaces @@ -341,8 +340,8 @@ public class ConcessioniOverMongoTest extends BasicServiceTestUnit{ c.getContentByPath(path).setTitolo("Pianta from "+key.replace(layerFolder.getAbsolutePath(),"")); c=update(c,target); c=upload(storage, target, c.getMongo_id(), path, - TestModel.getBaseFolder().list((file,name)->{return name.startsWith("pianta.shp");})); -// layers.get(key).toArray(new File[0])); +// TestModel.getBaseFolder().list((file,name)->{return name.startsWith("pianta.shp");})); + layers.get(key).toArray(new File[0])); } // Immagini diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/Caches.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/Caches.java new file mode 100644 index 0000000..ae86729 --- /dev/null +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/Caches.java @@ -0,0 +1,62 @@ +package org.gcube.application.geoportal.service.engine.caches; + +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.tests.TokenSetter; +import org.gcube.application.geoportal.service.engine.providers.AbstractScopedMap; +import org.gcube.application.geoportal.service.engine.providers.TTLObject; +import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertTrue; + +@Slf4j +public class Caches { + + @Test + public void testCache() throws ConfigurationException { + + TokenSetter.set("/gcube/devsec/devVRE"); + + Duration ttl=Duration.of(10, ChronoUnit.SECONDS); + + Duration monitorWindow=Duration.of(100, ChronoUnit.SECONDS); + + DummyCache cache= new DummyCache(); + cache.setTTL(ttl); + + + Instant startMonitoring=Instant.now(); + + LocalDateTime previous=cache.getObject(); + while(Duration.between(startMonitoring,Instant.now()).compareTo(monitorWindow)<0){ + LocalDateTime obj= cache.getObject(); + if(obj.equals(previous)){ + //objects are equals, TTL should be valid + // Assert : now-creationTime < TTL + assertTrue(Duration.between(obj,LocalDateTime.now()).compareTo(ttl)<0); + }else { + // different object only after TTL + // Assert : now-creationTime < TTL + assertTrue(Duration.between(obj,LocalDateTime.now()).compareTo(ttl)<0); + // Assert : now-previous.creationTime > TTL + assertTrue(Duration.between(previous,LocalDateTime.now()).compareTo(ttl)>0); + } + previous=obj; + try { + Thread.sleep(ttl.abs().dividedBy(2).toMillis()); + } catch (InterruptedException e) { + } + } + + + + + } + +} diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/DummyCache.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/DummyCache.java new file mode 100644 index 0000000..b8040b2 --- /dev/null +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/DummyCache.java @@ -0,0 +1,33 @@ +package org.gcube.application.geoportal.service.engine.caches; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.geoportal.service.engine.providers.AbstractScopedMap; +import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException; + +import java.time.LocalDateTime; + +@Slf4j +public class DummyCache extends AbstractScopedMap { + + + public DummyCache() { + super("Dummy cache"); + } + + @Override + protected LocalDateTime retrieveObject() throws ConfigurationException { + return LocalDateTime.now(); + } + + @Override + protected void dispose(LocalDateTime toDispose) { + } + + @Override + public void init() { + } + + + +} diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java index 1244c08..d4e6bbd 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/ws/DescribeWSFolder.java @@ -1,6 +1,7 @@ package org.gcube.application.geoportal.service.ws; import org.gcube.application.cms.tests.TokenSetter; +import org.gcube.application.geoportal.service.engine.WorkspaceManager; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.model.exceptions.StorageHubException; @@ -18,6 +19,8 @@ public class DescribeWSFolder { TokenSetter.set(context); shc= new StorageHubClient(); + shc.openVREFolder().get().getDescription(); + FolderContainer folder=shc.open(folderID).asFolder(); FolderItem item=folder.get(); diff --git a/geoportal-service/src/test/resources/logback.xml b/geoportal-service/src/test/resources/logback.xml index 7b7e4ee..8ec1f7c 100644 --- a/geoportal-service/src/test/resources/logback.xml +++ b/geoportal-service/src/test/resources/logback.xml @@ -8,7 +8,6 @@ - diff --git a/test-data/concessioni/packages/DATASET_GNA_01.csv b/test-data/concessioni/packages/DATASET_GNA_01.csv index 6ab4b49..464ca92 100644 --- a/test-data/concessioni/packages/DATASET_GNA_01.csv +++ b/test-data/concessioni/packages/DATASET_GNA_01.csv @@ -1,6 +1,5 @@ -"GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL","GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL/GNA_topografia_Aquileia Casa Bestie Ferite","GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL/GNA_topografia_Aquileia Casa Bestie Ferite" -"GNA_Castelseprio castrum-borgo_2019_rev_CdL","GNA_Castelseprio castrum-borgo_2019_rev_CdL/GNA_topografia_Castelseprio castrum-borgo","GNA_Castelseprio castrum-borgo_2019_rev_CdL/GNA_topografia_Castelseprio castrum-borgo" -"GNA_Monte Postale_2019_rev_CdL_FPDC","GNA_Monte Postale_2019_rev_CdL_FPDC/GNA_topografia_Monte Postale","GNA_Monte Postale_2019_rev_CdL_FPDC/GNA_topografia_Monte Postale" -"GNA_Castelseprio castrum-borgo_2019_rev_CdL","GNA_Castelseprio castrum-borgo_2019_rev_CdL/GNA_topografia_Castelseprio castrum-borgo","GNA_Castelseprio castrum-borgo_2019_rev_CdL/GNA_topografia_Castelseprio castrum-borgo" -"GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL","GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL/GNA_topografia_Aquileia Casa Bestie Ferite","GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL/GNA_topografia_Aquileia Casa Bestie Ferite" -"GNA_Appia Antica V miglio_2019_rev_CdL","GNA_Appia Antica V miglio_2019_rev_CdL/GNA_topografia_Appia Antica V miglio","GNA_Appia Antica V miglio_2019_rev_CdL/GNA_topografia_Appia Antica V miglio" \ No newline at end of file +"GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL","GNA_Aquileia Casa Bestie Ferite_2019_rev_CdL/GNA_topografia_Aquileia Casa Bestie Ferite/posizionamento saggi.shp" +"GNA_Castelseprio castrum-borgo_2019_rev_CdL","GNA_Castelseprio castrum-borgo_2019_rev_CdL/GNA_topografia_Castelseprio castrum-borgo/posizionamento saggi.shp" +"GNA_Monte Postale_2019_rev_CdL_FPDC","GNA_Monte Postale_2019_rev_CdL_FPDC/GNA_topografia_Monte Postale/pianta di fine scavo.shp" +"GNA_Castelseprio castrum-borgo_2019_rev_CdL","GNA_Castelseprio castrum-borgo_2019_rev_CdL/GNA_topografia_Castelseprio castrum-borgo/posizionamento saggi.shp" +"GNA_Appia Antica V miglio_2019_rev_CdL","GNA_Appia Antica V miglio_2019_rev_CdL/GNA_topografia_Appia Antica V miglio/posizionamento saggi.shp" \ No newline at end of file diff --git a/test-data/concessioni/packages/DATASET_GNA_02.csv b/test-data/concessioni/packages/DATASET_GNA_02.csv index 33fec22..c36a808 100644 --- a/test-data/concessioni/packages/DATASET_GNA_02.csv +++ b/test-data/concessioni/packages/DATASET_GNA_02.csv @@ -1,5 +1,4 @@ -"GNA_Bostel di Rotzo_Rev_FPDC","GNA_Bostel di Rotzo_Rev_FPDC/GNA_topografia_Bostel di Rotzo","GNA_Bostel di Rotzo_Rev_FPDC/GNA_topografia_Bostel di Rotzo" -"GNA_Braida Murada_Rev_FPDC","GNA_Braida Murada_Rev_FPDC/GNA_topografia_Braida Murada","GNA_Braida Murada_Rev_FPDC/GNA_topografia_Braida Murada" -"GNA_Egnazia_Rev_FPDC","GNA_Bostel di Rotzo_Rev_FPDC/GNA_topografia_Bostel di Rotzo","GNA_Bostel di Rotzo_Rev_FPDC/GNA_topografia_Bostel di Rotzo" -"GNA_Ferrandina_Rev_FPDC","GNA_Ferrandina_Rev_FPDC/GNA_topografia_Ferrandina","GNA_Ferrandina_Rev_FPDC/GNA_topografia_Ferrandina" -"GNA_Timpone della Motta_Rev_FPDC","GNA_Timpone della Motta_Rev_FPDC/GNA_topografia_Timpone della Motta","GNA_Timpone della Motta_Rev_FPDC/GNA_topografia_Timpone della Motta" \ No newline at end of file +"GNA_Bostel di Rotzo_Rev_FPDC","GNA_Bostel di Rotzo_Rev_FPDC/GNA_topografia_Bostel di Rotzo/posizionamento saggi.shp" +"GNA_Braida Murada_Rev_FPDC","GNA_Braida Murada_Rev_FPDC/GNA_topografia_Braida Murada/Posizionamento saggi.shp" +"GNA_Ferrandina_Rev_FPDC","GNA_Ferrandina_Rev_FPDC/GNA_topografia_Ferrandina/posizionamento saggi.shp" +"GNA_Timpone della Motta_Rev_FPDC","GNA_Timpone della Motta_Rev_FPDC/GNA_topografia_Timpone della Motta/posizionamento saggi.shp" \ No newline at end of file diff --git a/test-data/concessioni/packages/concessioni_23_04.csv b/test-data/concessioni/packages/concessioni_23_04.csv index 00af136..bc55ccd 100644 --- a/test-data/concessioni/packages/concessioni_23_04.csv +++ b/test-data/concessioni/packages/concessioni_23_04.csv @@ -1,4 +1,4 @@ -"GNA_Incoronata_2019_Rennes","/GNA_Incoronata_2019_Rennes/GNA_topografia_Incoronata/D","/GNA_Incoronata_2019_Rennes/GNA_topografia_Incoronata/E/SHP" -"MONTE MANNU_2019","/MONTE MANNU_2019/GNA_topografia_Monte_Mannu/D_Limiti dei saggi","/MONTE MANNU_2019/GNA_topografia_Monte_Mannu/E_Pianta di fine scavo" -"Santa Rosa Poviglio_2019","/Santa Rosa Poviglio_2019/GNA_Topografia_Poviglio/Posizionamento","/Santa Rosa Poviglio_2019/GNA_Topografia_Poviglio/Planimetrie" -"Usini Tomestighes_integrata","/Usini Tomestighes_integrata/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/GNA_Tomestighes_2019/D_GNA_Posizionamento_limiti_aree_indagate","/Usini Tomestighes_integrata/GNA_TOMESTIGHES_2019/GNA_Topografia_Tomestighes/GNA_Tomestighes_2019/E_GNA_Piante_fine_scavo_Tomestighes/Area A est" \ No newline at end of file +"GNA_Incoronata_2019_Rennes","GNA_Incoronata_2019_Rennes/GNA_topografia_Incoronata/D/limites.shp" +"MONTE MANNU_2019","MONTE MANNU_2019/GNA_topografia_Monte_Mannu/D_Limiti dei saggi/Limiti dei saggi_polyg.shp" +"Santa Rosa Poviglio_2019","Santa Rosa Poviglio_2019/GNA_Topografia_Poviglio/Posizionamento/PosizionamentoSettori15-19.shp" +"Usini Tomestighes_integrata","Usini Tomestighes_integrata/integrazioni/GNA_Tomestighes_2019/D_GNA_Posizionamento_limiti_aree_ricognizione/area di concessione.shp" \ No newline at end of file diff --git a/test-data/concessioni/packages/invio_08_05.csv b/test-data/concessioni/packages/invio_08_05.csv index 8dc9e6a..92df983 100644 --- a/test-data/concessioni/packages/invio_08_05.csv +++ b/test-data/concessioni/packages/invio_08_05.csv @@ -1,7 +1,7 @@ -"Farnese (VT)_Sorgenti della Nova_documentazione fine scavo 2019","Farnese (VT)_Sorgenti della Nova_documentazione fine scavo 2019/Gis Sorgenti della Nova","Farnese (VT)_Sorgenti della Nova_documentazione fine scavo 2019/Gis Sorgenti della Nova/Piante di fase" -"GNA_Aquileia_ExPasqualis_ok","GNA_Aquileia_ExPasqualis_ok/GNA_topografia_Aquileia ex Pasqualis_2019","GNA_Aquileia_ExPasqualis_ok/GNA_topografia_Aquileia ex Pasqualis_2019" -"Jesolo (Ve) - Loc. San Mauro e Torre del Caligo","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/topografia_modificata","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/topografia_modificata" -"Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va","Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va/CONSEGNA_WGS84","Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va/CONSEGNA_WGS84" -"Montecompatri_Tenuta_Castiglione_doc_IVa_2019","Montecompatri_Tenuta_Castiglione_doc_IVa_2019/GNA_topografia_Gabii_Louvre/POSIZIONAMENTO","Montecompatri_Tenuta_Castiglione_doc_IVa_2019/GNA_topografia_Gabii_Louvre/PIANTA FINE SCAVO" -"SAN_CASCIANO_BAGNI","SAN_CASCIANO_BAGNI/POSIZIONAMENTO_EPSG_4326","SAN_CASCIANO_BAGNI" -"Tarquinia_Civita_GNA_2019","Tarquinia_Civita_GNA_2019/Va_D__Topografia_Tarquinia/Posizionamento","Tarquinia_Civita_GNA_2019/Va_D__Topografia_Tarquinia/Posizionamento" \ No newline at end of file +"Farnese (VT)_Sorgenti della Nova_documentazione fine scavo 2019","Farnese (VT)_Sorgenti della Nova_documentazione fine scavo 2019/Gis Sorgenti della Nova/FN19_generalplan WSG84 EPSG4326 sector XIII polilinea.shx" +"GNA_Aquileia_ExPasqualis_ok","GNA_Aquileia_ExPasqualis_ok/GNA_topografia_Aquileia ex Pasqualis_2019/AQU19PSQ_limiti_saggi_poligoni.shp" +"Jesolo (Ve) - Loc. San Mauro e Torre del Caligo","Jesolo (Ve) - Loc. San Mauro e Torre del Caligo/topografia_modificata/posizionamento_vb.shp" +"Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va","Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va/CONSEGNA_WGS84/GPR_area_WGS84.shp" +"Montecompatri_Tenuta_Castiglione_doc_IVa_2019","Montecompatri_Tenuta_Castiglione_doc_IVa_2019/GNA_topografia_Gabii_Louvre/POSIZIONAMENTO/Gabii_Louvre_posizionamento_EPSG4326.shp" +"SAN_CASCIANO_BAGNI","SAN_CASCIANO_BAGNI/POSIZIONAMENTO_EPSG_4326/Bagno_Grande_posizionamento_4326_prova.shp" +"Tarquinia_Civita_GNA_2019","Tarquinia_Civita_GNA_2019/Va_D__Topografia_Tarquinia/Posizionamento/Posizionamento dei limiti.shp" \ No newline at end of file diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/StressTest.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/StressTest.java new file mode 100644 index 0000000..eb23ef1 --- /dev/null +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/StressTest.java @@ -0,0 +1,54 @@ +package org.gcube.application.cms.usecases; + +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.cms.tests.TokenSetter; +import org.gcube.application.cms.tests.model.TestModel; +import org.gcube.application.geoportal.common.model.legacy.Concessione; +import org.gcube.application.geoportal.common.rest.MongoConcessioni; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.gcube.application.geoportal.client.GeoportalAbstractPlugin.mongoConcessioni; + +@Slf4j +public class StressTest { + + + private static ExecutorService service = Executors.newFixedThreadPool(100); + + public static void main(String[] args) { + TokenSetter.set("/gcube/devsec/devVRE"); + AtomicLong executed=new AtomicLong(0); + + int numRequests=10000; + for(int i=0;i error=new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue success=new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue warning=new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue noReport=new ConcurrentLinkedQueue<>(); + } + + private static ConcessionPublisherThread.Report report=new ConcessionPublisherThread.Report(); + + private static ExecutorService exec=Executors.newFixedThreadPool(5); + + public static void submit(ConcessionPublisherThread c){ + log.info("Submitting {}",c); + exec.submit(c); + log.info("Submitted {} ",queued.incrementAndGet()); + } + + public static ConcessionPublisherThread.Report waitCompletion(){ + try { + while (!exec.awaitTermination(2, TimeUnit.MINUTES)) { + log.info("Waiting .. completed {}, out of {} ",completed.get(),queued.get()); + if(completed.get()==queued.get()) exec.shutdown(); + } + }catch(InterruptedException e){ + log.info("Process finished"); + } + return report; + } + + @NonNull + private File baseDir; + @NonNull + private String positionPath; + @NonNull + private String projectName; + + + + + + @Override + public void run() { + Concessione c=null; + try { + MongoConcessioni client=mongoConcessioni().build(); + StorageUtils storage=new StorageUtils(); + + //NB raggruppa per file + Map.Entry> posSets = Files.getAllShapeSet(new File(baseDir,positionPath),true). + entrySet().stream().findFirst().get(); + Map> pianteSets = Files.getAllShapeSet(new File(baseDir,projectName),true); + + if(pianteSets.size()>1) + pianteSets.remove(posSets.getKey()); + + + + log.debug("Entry {} pos Size {} piante {} ",projectName,posSets.getValue().size(),pianteSets.size()); + + + c = createMock(projectName,baseDir.getName(), pianteSets, posSets.getValue(), client, storage); + + switch(c.getReport().getStatus()){ + case PASSED: report.getSuccess().add(c); + break; + case ERROR: report.getError().add(c); + break; + case WARNING:report.getWarning().add(c); + break; + } + + }catch(Throwable t){ + log.error("Problematic entry "+this,t); + report.getNoReport().add(c); + }finally{ + log.info("Completed N {}", completed.incrementAndGet()); + } + } + + + + private static Concessione createMock(String baseName, String packageName, Map> piante, List pos, + MongoConcessioni client, StorageUtils storage) throws Exception { + + Concessione c= TestModel.prepareConcessione(piante.size(), 1); + c.setNome("Mock for "+baseName+" ("+packageName+")"); + c= client.createNew(c); + String mongoId=c.getMongo_id(); + + // TEST DATA, DO NOT CARE + client.registerFileSet(mongoId, FileSets.prepareRequest(storage, + Concessione.Paths.RELAZIONE,new File (TestModel.getBaseFolder(),"relazione.pdf"))); + + client.registerFileSet(mongoId, FileSets.prepareRequest(storage, + Concessione.Paths.imgByIndex(0),Files.getSiblings(TestModel.getBaseFolder(),"immagine").get(0))); + + // POSIZIONAMENTO + + client.registerFileSet(mongoId, FileSets.prepareRequest(storage, + Concessione.Paths.POSIZIONAMENTO,pos.toArray(new File[pos.size()]))); + + // PIANTE + Map.Entry>[] entries= piante.entrySet().toArray(new Map.Entry[0]); + for( int i= 0; i< piante.size();i++) { + // Set layer name + c=client.getById(mongoId); + String path=Concessione.Paths.piantaByIndex(i); + c.getContentByPath(path).setTitolo(" Pianta from "+entries[i].getKey()); + client.update(mongoId, Serialization.write(c)); + + //Set fileset + client.registerFileSet(mongoId, FileSets.prepareRequest(storage,path, entries[i].getValue().toArray(new File[0]))); + } + + c=client.publish(mongoId); + + System.out.println("@@@ Concessione "+c.getNome()+"\t STATUS : "+ c.getReport().getStatus()); + + return c; + } +} diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java index e25f6d4..ca59625 100644 --- a/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java @@ -12,9 +12,11 @@ import org.gcube.application.geoportal.common.model.legacy.report.ValidationRepo import org.gcube.application.geoportal.common.rest.MongoConcessioni; import org.gcube.application.geoportal.common.utils.Files; import org.gcube.application.geoportal.common.utils.StorageUtils; +import org.gcube.contentmanagement.blobstorage.report.Report; import java.io.*; import java.util.*; +import java.util.function.Consumer; import static org.gcube.application.geoportal.client.GeoportalAbstractPlugin.mongoConcessioni; import static org.junit.Assert.assertTrue; @@ -27,137 +29,77 @@ public class MockFromFolder { String context= "/gcube/devsec/devVRE"; File descriptorsBaseFolder=new File(TestModel.getBaseFolder(),"packages"); - //Concessioni 04-03 - String packageBaseDir="/Users/fabioisti/Documents/Concessioni 04-03/"; - String csvDescriptor="concessioni04-03_filtered.csv"; - //DATASET_GNA_01 -// String packageBaseDir="/Users/fabioisti/Documents/DATASET_GNA_01"; -// String csvDescriptor="DATASET_GNA_01.csv"; - - //DATASET_GNA_02 - //String packageBaseDir="/Users/fabioisti/Documents/DATASET_GNA_02"; - //String csvDescriptor="DATASET_GNA_02.csv"; + // config csv -> base folder + Map configurations=new HashMap<>(); + //Concessioni 04-03 + configurations.put("concessioni04-03.csv","/Users/fabioisti/Documents/Concessioni 04-03/"); + //DATASET_GNA_01 + configurations.put("DATASET_GNA_01.csv","/Users/fabioisti/Documents/DATASET_GNA_01"); + //DATASET_GNA_02 + configurations.put("DATASET_GNA_02.csv","/Users/fabioisti/Documents/DATASET_GNA_02"); // invio_08_02 -// String packageBaseDir="/Users/fabioisti/Documents/invio_08_05"; -// String csvDescriptor="invio_08_05.csv"; - + configurations.put("invio_08_05.csv","/Users/fabioisti/Documents/invio_08_05"); // concessioni 23_04 -// String packageBaseDir="/Users/fabioisti/Documents/Concessioni_23_04"; -// String csvDescriptor="concessioni_23_04.csv"; - - - + configurations.put("concessioni_23_04.csv","/Users/fabioisti/Documents/Concessioni_23_04"); TokenSetter.set(context); - MongoConcessioni client=mongoConcessioni().build(); - StorageUtils storage=new StorageUtils(); + long start= System.currentTimeMillis(); + for(Map.Entry entry : configurations.entrySet()){ + String packageBaseDir=entry.getValue(); + String csvDescriptor=entry.getKey(); + System.out.println("Reading "+csvDescriptor); - long publishedCount=0l; - long successcount=0l; - long entrycount=0l; - //Parse CSV descriptor - File baseDir=new File(packageBaseDir); - ArrayList pushed=new ArrayList<>(); + File baseDir=new File(packageBaseDir); - CSVReader reader = new CSVReader(new FileReader(new File(descriptorsBaseFolder,csvDescriptor))); - String [] nextLine; - //reads one line at a time - while ((nextLine = reader.readNext()) != null) - { - entrycount++; - //Create new + CSVReader reader = new CSVReader(new FileReader(new File(descriptorsBaseFolder,csvDescriptor))); + String [] nextLine; + //reads one line at a time + while ((nextLine = reader.readNext()) != null) { String projectName = nextLine[0]; String positionPath = nextLine[1]; - - - try { - //NB raggruppa per file - Map.Entry> posSets = Files.getAllShapeSet(new File(baseDir,positionPath),true). - entrySet().stream().findFirst().get(); - Map> pianteSets = Files.getAllShapeSet(new File(baseDir,projectName),true); - - if(pianteSets.size()>1) - pianteSets.remove(posSets.getKey()); - - - - log.debug("Entry {} pos Size {} piante {} ",projectName,posSets.getValue().size(),pianteSets.size()); - - - Concessione c = createMock(projectName,baseDir.getName(), pianteSets, posSets.getValue(), client, storage); - - publishedCount++; - if (c.getReport().getStatus().equals(ValidationReport.ValidationStatus.PASSED)) - successcount++; - pushed.add(c); - - - }catch(Throwable t){ - System.err.println("Problematic entry "+projectName); - t.printStackTrace(System.err); + ConcessionPublisherThread.submit(new ConcessionPublisherThread(baseDir,positionPath,projectName)); } } - - System.out.println("Done "+publishedCount+" [SUCCESS : "+successcount+"] \t OUT OF :"+entrycount+" entries"); - - pushed.forEach(c -> { - try{ - System.out.println(c.getNome()+"\t"+c.getMongo_id()+"\t"+c.getReport().getStatus()); - }catch(Throwable t){ - System.out.println(c.getNome()+"\t"+c.getMongo_id()+"\t PROBLEMATIC, NO REPORT"); - } - }); - } - + System.out.println("Waiting.. "); + ConcessionPublisherThread.Report report =ConcessionPublisherThread.waitCompletion(); + System.out.println("Completed in "+(System.currentTimeMillis()-start)+"ms"); - private static Concessione createMock(String baseName,String packageName,Map> piante, List pos, - MongoConcessioni client, StorageUtils storage) throws Exception { - - Concessione c= TestModel.prepareConcessione(piante.size(), 2); - c.setNome("Mock for "+baseName+" ("+packageName+")"); - c= client.createNew(c); - String mongoId=c.getMongo_id(); - - // TEST DATA, DO NOT CARE - client.registerFileSet(mongoId, FileSets.prepareRequest(storage, - Concessione.Paths.RELAZIONE,new File (TestModel.getBaseFolder(),"relazione.pdf"))); - - client.registerFileSet(mongoId, FileSets.prepareRequest(storage, - Concessione.Paths.imgByIndex(0),new File(TestModel.getBaseFolder(),"immagine.png"))); - - // POSIZIONAMENTO + System.out.println("SUCCESS "+report.getSuccess().size()); + report.getSuccess().forEach(concessionePrinter); + System.out.println("ERROR "+report.getError().size()); + report.getError().forEach(concessionePrinter); + System.out.println("WARNING "+report.getWarning().size()); + report.getWarning().forEach(concessionePrinter); + System.out.println("NO REPORT "+report.getNoReport().size()); + report.getNoReport().forEach(concessionePrinter); + } - client.registerFileSet(mongoId, FileSets.prepareRequest(storage, - Concessione.Paths.POSIZIONAMENTO,pos.toArray(new File[pos.size()]))); - // PIANTE - Map.Entry>[] entries= piante.entrySet().toArray(new Map.Entry[0]); - for( int i= 0; i< piante.size();i++) { - // Set layer name - c=client.getById(mongoId); - String path=Concessione.Paths.piantaByIndex(i); - c.getContentByPath(path).setTitolo(" Pianta from "+entries[i].getKey()); - client.update(mongoId, Serialization.write(c)); - //Set fileset - client.registerFileSet(mongoId, FileSets.prepareRequest(storage,path, entries[i].getValue().toArray(new File[0]))); + private static Consumer concessionePrinter=new Consumer() { + @Override + public void accept(Concessione c) { + try { + System.out.println(c.getNome() + "\t" + c.getMongo_id() + "\t" + c.getReport().getStatus()); + } catch (Throwable t) { +// System.out.println(c.getNome()+"\t"+c.getMongo_id()+"\t PROBLEMATIC, NO REPORT"); + throw t; + } } + }; + - c=client.publish(mongoId); - System.out.println("@@@ Concessione "+c.getNome()+"\t STATUS : "+ c.getReport().getStatus()); - return c; - } } diff --git a/use-cases/src/test/resources/logback.xml b/use-cases/src/test/resources/logback.xml new file mode 100644 index 0000000..b84c76b --- /dev/null +++ b/use-cases/src/test/resources/logback.xml @@ -0,0 +1,17 @@ + + + + + + %d{HH:mm:ss} | %logger{0}:%L - %msg%n + + + + + + + + + + + \ No newline at end of file