From 9eab028bab6d468683402f904053437c4fca6755 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Wed, 6 Oct 2021 18:26:10 +0200 Subject: [PATCH] Downgrade to legacy storage implementation --- .../geoportal/common/utils/StorageUtils.java | 4 +- .../geoportal/common/StorageUtilsTest.java | 101 ++++++++++++++++++ .../geoportal/common/TokenSetter.java | 33 ++++++ .../providers/StorageClientProvider.java | 21 ++-- .../service/engine/caches/Caches.java | 45 ++++++++ pom.xml | 5 +- .../mocks/ConcessionPublisherThread.java | 4 +- .../cms/usecases/mocks/MockFromFolder.java | 4 +- 8 files changed, 195 insertions(+), 22 deletions(-) create mode 100644 geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java create mode 100644 geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java index 9e4bf02..6f383b1 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java @@ -19,7 +19,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class StorageUtils { - public static final IClient getClient(){ + private static final IClient getClient(){ return new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME, ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient(); } @@ -30,7 +30,7 @@ public class StorageUtils { } public void forceClose(){ - client.forceClose(); +// client.forceClose(); } public TempFile putOntoStorage(InputStream source,String filename) throws RemoteBackendException, FileNotFoundException{ diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java new file mode 100644 index 0000000..23235fd --- /dev/null +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/StorageUtilsTest.java @@ -0,0 +1,101 @@ +package org.gcube.application.geoportal.common; + +import com.mongodb.MongoWaitQueueFullException; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.geoportal.common.rest.InterfaceConstants; +import org.gcube.application.geoportal.common.utils.ContextUtils; +import org.gcube.application.geoportal.common.utils.StorageUtils; +import org.gcube.contentmanagement.blobstorage.service.IClient; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.MemoryType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; +import org.junit.Before; +import org.junit.Test; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +public class StorageUtilsTest { + + IClient client= null; + + @Before + public void init(){ + TokenSetter.set("/gcube/devsec/devVRE"); + client=new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME, + ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient(); + } + + private String getFileID() throws FileNotFoundException { + return client.put(true).LFile( + new FileInputStream("../test-data/concessioni/relazione.pdf")). + RFile(StorageUtils.getUniqueString()); + } + + private String getURL(String id){ + return client.getHttpsUrl().RFile(id); + } + + @Test + public void testParallelStorage() throws FileNotFoundException, InterruptedException { + + ExecutorService service = Executors.newFixedThreadPool(10); + LocalDateTime start=LocalDateTime.now(); + AtomicLong executed = new AtomicLong(0); + AtomicLong launched = new AtomicLong(0); + + String id=getFileID(); + //for 100 secs + while(Duration.between(start,LocalDateTime.now()). + compareTo(Duration.of(100, ChronoUnit.SECONDS))<0){ + service.execute(new Runnable() { + @Override + public void run() { + try { + System.out.println(getURL(id)); + } catch (Throwable t) { + log.info("Too many connections... "); + }finally{ + executed.incrementAndGet(); + try {Thread.sleep(500);} catch (InterruptedException i) {} + } + } + }); + launched.incrementAndGet(); + } + + while (!service.awaitTermination(2, TimeUnit.MINUTES)) { + log.info("Waiting .. completed {}, out of {} ",executed.get(),launched.get()); + if(executed.get()==launched.get()) service.shutdown(); + } + } + + @Test + public void testSerialStorage() throws FileNotFoundException { + TokenSetter.set("/gcube/devsec/devVRE"); + //get client + client=new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME, + ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient(); + + //put file + String id= client.put(true).LFile( + new FileInputStream("../test-data/concessioni/relazione.pdf")). + RFile(StorageUtils.getUniqueString()); + + for(int i = 0; i<1000;i++){ + //get ID + System.out.println(client.getHttpsUrl().RFile(id)); + } + } + + + +} diff --git a/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java new file mode 100644 index 0000000..b8d27bd --- /dev/null +++ b/geoportal-common/src/test/java/org/gcube/application/geoportal/common/TokenSetter.java @@ -0,0 +1,33 @@ +package org.gcube.application.geoportal.common; + +import lombok.extern.slf4j.Slf4j; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; + +import java.util.Properties; + +@Slf4j +public class TokenSetter { + + + private static Properties props=new Properties(); + + static{ + try { + props.load(TokenSetter.class.getResourceAsStream("/tokens.properties")); + } catch (Exception e) { + throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION"); + } + } + + + public static void set(String scope){ + try{ + if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope); + SecurityTokenProvider.instance.set(props.getProperty(scope)); + }catch(Throwable e){ + log.warn("Unable to set token for scope "+scope,e); + } + ScopeProvider.instance.set(scope); + } +} diff --git a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java index eaaf08a..a205727 100644 --- a/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java +++ b/geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java @@ -1,23 +1,10 @@ package org.gcube.application.geoportal.service.engine.providers; import lombok.extern.slf4j.Slf4j; -import org.gcube.application.geoportal.common.rest.InterfaceConstants; + import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException; -import org.gcube.application.geoportal.service.utils.ContextUtils; -import org.gcube.contentmanagement.blobstorage.service.IClient; -import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; -import org.gcube.contentmanager.storageclient.wrapper.AccessType; -import org.gcube.contentmanager.storageclient.wrapper.MemoryType; -import org.gcube.contentmanager.storageclient.wrapper.StorageClient; -import org.gcube.data.transfer.library.utils.Utils; -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.time.Duration; -import java.time.temporal.ChronoUnit; @Slf4j public class StorageClientProvider extends AbstractScopedMap { @@ -30,7 +17,11 @@ public class StorageClientProvider extends AbstractScopedMap { @Override protected StorageUtils retrieveObject() throws ConfigurationException { - return new StorageUtils(); + try{ + return new StorageUtils(); + }catch(Throwable t){ + throw new ConfigurationException("unable to get Storage",t); + } // return new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME, ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient(); } diff --git a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/Caches.java b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/Caches.java index 296e7f6..95b29d3 100644 --- a/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/Caches.java +++ b/geoportal-service/src/test/java/org/gcube/application/geoportal/service/engine/caches/Caches.java @@ -1,15 +1,21 @@ package org.gcube.application.geoportal.service.engine.caches; +import ch.qos.logback.core.net.SyslogOutputStream; import com.mongodb.MongoWaitQueueFullException; import lombok.extern.slf4j.Slf4j; import org.gcube.application.cms.tests.TokenSetter; +import org.gcube.application.cms.tests.model.TestModel; +import org.gcube.application.geoportal.common.utils.StorageUtils; import org.gcube.application.geoportal.service.BasicServiceTestUnit; +import org.gcube.application.geoportal.service.engine.ImplementationProvider; import org.gcube.application.geoportal.service.engine.mongo.ConcessioniMongoManager; import org.gcube.application.geoportal.service.engine.providers.AbstractScopedMap; import org.gcube.application.geoportal.service.engine.providers.TTLObject; import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException; import org.junit.Test; +import java.io.File; +import java.io.FileNotFoundException; import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; @@ -98,5 +104,44 @@ public class Caches extends BasicServiceTestUnit { } + @Test + public void testStorageCache() throws ConfigurationException, FileNotFoundException, InterruptedException { + TokenSetter.set(scope); + ExecutorService service = Executors.newFixedThreadPool(10); + LocalDateTime start=LocalDateTime.now(); + AtomicLong executed = new AtomicLong(0); + AtomicLong launched = new AtomicLong(0); + + final StorageUtils storage=ImplementationProvider.get().getStorageProvider().getObject(); + + String id =storage.putOntoStorage(new File(TestModel.getBaseFolder(),"relazione.pdf"))[0].getId(); + + //for 100 secs + while(Duration.between(start,LocalDateTime.now()). + compareTo(Duration.of(100, ChronoUnit.SECONDS))<0){ + service.execute(new Runnable() { + @Override + public void run() { + try { +// System.out.println(ImplementationProvider.get().getStorageProvider().getObject().getURL(id)); + storage.getURL(id); +// } catch (ConfigurationException e) { +// e.printStackTrace(); + } catch (MongoWaitQueueFullException e) { + log.info("Too many connections... "); + }finally{ + executed.incrementAndGet(); + try {Thread.sleep(500);} catch (InterruptedException i) {} + } + } + }); + launched.incrementAndGet(); + } + + while (!service.awaitTermination(2, TimeUnit.MINUTES)) { + log.info("Waiting .. completed {}, out of {} ",executed.get(),launched.get()); + if(executed.get()==launched.get()) service.shutdown(); + } + } } diff --git a/pom.xml b/pom.xml index 05d5a63..2bec794 100644 --- a/pom.xml +++ b/pom.xml @@ -107,12 +107,13 @@ org.gcube.contentmanagement storage-manager-core - [2.0.0, 3.0.0-SNAPSHOT) + + [2.0.0, 2.9.0-SNAPSHOT) org.gcube.contentmanagement storage-manager-wrapper - [2.0.0, 3.0.0-SNAPSHOT) + [2.0.0, 2.9.0-SNAPSHOT) diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/ConcessionPublisherThread.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/ConcessionPublisherThread.java index 3ed3769..a9ee8c0 100644 --- a/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/ConcessionPublisherThread.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/ConcessionPublisherThread.java @@ -37,7 +37,7 @@ public class ConcessionPublisherThread implements Runnable{ private ConcurrentLinkedQueue error=new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue success=new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue warning=new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue noReport=new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue noReport=new ConcurrentLinkedQueue<>(); } private static ConcessionPublisherThread.Report report=new ConcessionPublisherThread.Report(); @@ -106,7 +106,7 @@ public class ConcessionPublisherThread implements Runnable{ }catch(Throwable t){ log.error("Problematic entry "+this,t); - report.getNoReport().add(c); + report.getNoReport().add(projectName + baseDir); }finally{ log.info("Completed N {}", completed.incrementAndGet()); } diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java index ca59625..fbfe307 100644 --- a/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/mocks/MockFromFolder.java @@ -81,7 +81,9 @@ public class MockFromFolder { System.out.println("WARNING "+report.getWarning().size()); report.getWarning().forEach(concessionePrinter); System.out.println("NO REPORT "+report.getNoReport().size()); - report.getNoReport().forEach(concessionePrinter); + report.getNoReport().forEach(s -> { + System.out.println(s); + }); }