gCubeSystem
/
gcube-cms-suite
14
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Uniform Storage access

This commit is contained in:
Fabio Sinibaldi 2021-09-27 15:23:03 +02:00
parent 42f3b14a8f
commit 0eb83ef6a8
11 changed files with 219 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
geoportal-client/pom.xml
View File

@ -95,7 +95,6 @@
</dependency>
</dependencies>
</project>

14
geoportal-client/src/test/java/org/gcube/application/geoportal/StorageTests.java
View File

@ -18,7 +18,7 @@ import java.net.URL;
public class StorageTests {
public static void main(String[] args) throws IOException {
String filename= "relazione.pdf";
// String filename= "relazione.pdf";
// String sourceContext="/d4science.research-infrastructures.eu";
// String targetContext="/pred4s/preprod/preVRE";
@ -29,16 +29,20 @@ public class StorageTests {
// StorageHubClient sgClient=new StorageHubClient();
// System.out.println(new StorageUtils().getUrlById(file.getId()));
TokenSetter.set(targetContext);
System.out.println(new StorageUtils().getURL("e6b5d48d-b37d-4280-900d-f407a1ddc7d3"));
System.out.println(new File(".").getAbsolutePath());
TempFile file= new StorageUtils().putOntoStorage(new File(TestModel.getBaseFolder(),filename),filename);
IOUtils.copy(new URL(StorageUtils.getUrlById(file.getId())).openStream(),System.out);
// System.out.println(new File(".").getAbsolutePath());
// TempFile file= new StorageUtils().putOntoStorage(new File(TestModel.getBaseFolder(),filename),filename);
//
// IOUtils.copy(new URL(StorageUtils.getUrlById(file.getId())).openStream(),System.out);
}
}

27
geoportal-client/src/test/java/org/gcube/application/geoportal/clients/StatefulClientTests.java
View File

@ -50,7 +50,7 @@ public class StatefulClientTests extends BasicVreTests{
//Alternative Method
InputStreamDescriptor isDesc=
new InputStreamDescriptor(new FileInputStream(new File(TestModel.getBaseFolder(),
"immagine.png")), "immagine.png");
"immagine.png")), "San Mauro_drone totale.JPG");
manager.addImmagineRappresentativa(toRegisterImg, isDesc);
@ -67,13 +67,17 @@ public class StatefulClientTests extends BasicVreTests{
// Posizionamento scavo
manager.setPosizionamento(source.getPosizionamentoScavo(),
storage.putOntoStorage(new File(TestModel.getBaseFolder(),"pos.shp"), "pos.shp"));
storage.putOntoStorage(
new File(TestModel.getBaseFolder(),"pos.shp"),"pos.shp"));
// Piante
manager.addPiantaFineScavo(source.getPianteFineScavo().get(0),
storage.putOntoStorage(new File(TestModel.getBaseFolder(),"pianta.shp"), "pianta.shp"),
storage.putOntoStorage(new File(TestModel.getBaseFolder(),"pianta.shx"), "pianta.shx"));
storage.putOntoStorage(
new File(TestModel.getBaseFolder(),"pianta.shp"),"pianta.shp"));
// new File("/Users/fabioisti/Documents/invio_08_05/Montalto di Castro (VT)_Vulci_Indagini non invasive_Doc. paragr._Va/CONSEGNA_WGS84")
// .listFiles((file,name)->{return name.startsWith("Mag_anomalies_WGS84");})));
return manager.publish();
}
@ -111,18 +115,5 @@ public class StatefulClientTests extends BasicVreTests{
manager.delete();
}
@Test
public void replace() {
}
@Test
public void getById() {
}
@Test
public void list() {
}
}

182
geoportal-client/src/test/java/org/gcube/application/geoportal/clients/StatelessClientTests.java
View File

@ -2,6 +2,7 @@ package org.gcube.application.geoportal.clients;
import static org.gcube.application.geoportal.client.GeoportalAbstractPlugin.mongoConcessioni;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.FileInputStream;
@ -28,93 +29,6 @@ public class StatelessClientTests extends BasicVreTests{
private MongoConcessioni client=mongoConcessioni().build();
@Test
public void testCreateNew() throws Exception {
Concessione c= client.createNew(TestModel.prepareEmptyConcessione());
assertNotNull(c);
assertNotNull(c.getMongo_id());
}
@Test
public void testDeleteById() throws Exception {
Concessione c= client.createNew(TestModel.prepareEmptyConcessione());
client.deleteById(c.getMongo_id());
}
@Test
public void testList() throws Exception {
final AtomicLong counter=new AtomicLong();
long before=System.currentTimeMillis();
client.getList().forEachRemaining((Concessione c)-> {counter.addAndGet(1);});
System.out.println("Loaded "+counter+" in "+(System.currentTimeMillis()-before)+" ms");
}
@Test
public void testReplace() throws Exception {
Concessione c= client.createNew(TestModel.prepareEmptyConcessione());
String title="My new shiny Title";
c.setNome(title);
Concessione c1=client.replace(c);
assertEquals(title, c1.getNome());
}
@Test
public void testUploadFileSet() throws Exception {
Concessione c= client.createNew(TestModel.prepareConcessione());
assertNotNull(c.getRelazioneScavo());
AddSectionToConcessioneRequest request=
FileSets.prepareRequest(new StorageUtils(),Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf"));
c= client.registerFileSet(c.getMongo_id(), request);
assertNotNull(c.getRelazioneScavo().getActualContent());
assertNotNull(c.getRelazioneScavo().getActualContent().get(0));
}
@Test
public void testPublsh() throws Exception {
publish(true);
}
public Concessione publish(Boolean verify) throws Exception {
return publish(verify,client);
}
public static Concessione publish(Boolean verify,MongoConcessioni client) throws Exception {
Concessione c= client.createNew(TestModel.prepareConcessione(1,1));
StorageUtils storage = new StorageUtils();
String mongoId=c.getMongo_id();
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.imgByIndex(0),new File(TestModel.getBaseFolder(),"immagine.png")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.POSIZIONAMENTO,new File(TestModel.getBaseFolder(),"pos.shp")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.piantaByIndex(0),new File(TestModel.getBaseFolder(),"pianta.shp")));
c=client.publish(mongoId);
if(verify)
assertTrue(c.getReport().getStatus().equals(ValidationStatus.PASSED));
return c;
}
@Test
public void getConfiguration() throws Exception {
@ -167,23 +81,99 @@ public class StatelessClientTests extends BasicVreTests{
// }
}
@Test
public void testList() throws Exception {
final AtomicLong counter=new AtomicLong();
long before=System.currentTimeMillis();
client.getList().forEachRemaining((Concessione c)-> {counter.addAndGet(1);});
System.out.println("Loaded "+counter+" in "+(System.currentTimeMillis()-before)+" ms");
}
@Test
public void testCreateNew() throws Exception {
Concessione c= client.createNew(TestModel.prepareEmptyConcessione());
assertNotNull(c);
assertNotNull(c.getMongo_id());
}
@Test
public void testReplace() throws Exception {
Concessione testObject= client.createNew(TestModel.prepareEmptyConcessione());
String title="My new shiny Title";
testObject.setNome(title);
Concessione c1=client.replace(testObject);
assertEquals(title, c1.getNome());
}
@Test
public void testUploadFileSet() throws Exception {
Concessione testObject= client.createNew(TestModel.prepareEmptyConcessione());
AddSectionToConcessioneRequest request=
// FileSets.prepareRequest(new StorageUtils(),Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf"));
FileSets.build(Paths.ABSTRACT_RELAZIONE).add(
new StorageUtils().putOntoStorage(new File(TestModel.getBaseFolder(),"relazione.pdf"),"San Mauro_drone totale.JPG"))
.getTheRequest();
testObject= client.registerFileSet(testObject.getMongo_id(), request);
assertNotNull(testObject.getContentByPath(Paths.ABSTRACT_RELAZIONE).getActualContent());
assertNotNull(testObject.getContentByPath(Paths.ABSTRACT_RELAZIONE).getActualContent().get(0));
}
@Test
public void testDeleteById() throws Exception {
Concessione c= client.createNew(TestModel.prepareEmptyConcessione());
client.deleteById(c.getMongo_id());
}
@Test
public Concessione testPublsh() throws Exception {
Concessione c= client.createNew(TestModel.prepareConcessione(1,1));
StorageUtils storage = new StorageUtils();
String mongoId=c.getMongo_id();
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.RELAZIONE,new File(TestModel.getBaseFolder(),"relazione.pdf")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.imgByIndex(0),new File(TestModel.getBaseFolder(),"immagine.png")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.POSIZIONAMENTO,new File(TestModel.getBaseFolder(),"pos.shp")));
client.registerFileSet(mongoId,
FileSets.prepareRequest(storage,Paths.piantaByIndex(0),new File(TestModel.getBaseFolder(),"pianta.shp")));
c=client.publish(mongoId);
assertTrue(c.getReport().getStatus().equals(ValidationStatus.PASSED));
return c;
}
@Test
public void testCleanFileSet() throws Exception {
Concessione c=publish(false);
client.unPublish(c.getMongo_id());
Concessione testObject=testPublsh();
//Precheck to be sure
assertFalse(c.getPosizionamentoScavo().getActualContent().isEmpty());
assertFalse(c.getPianteFineScavo().get(0).getActualContent().isEmpty());
assertFalse(testObject.getPosizionamentoScavo().getActualContent().isEmpty());
assertFalse(testObject.getPianteFineScavo().get(0).getActualContent().isEmpty());
//Clear pos
c=client.cleanFileSet(c.getMongo_id(),Paths.POSIZIONAMENTO);
assertTrue(c.getPosizionamentoScavo().getActualContent().isEmpty());
testObject=client.cleanFileSet(testObject.getMongo_id(),Paths.POSIZIONAMENTO);
assertTrue(testObject.getPosizionamentoScavo().getActualContent().isEmpty());
//Clear pianta [0]
c=client.cleanFileSet(c.getMongo_id(),Paths.piantaByIndex(0));
assertTrue(c.getPianteFineScavo().get(0).getActualContent().isEmpty());
testObject =client.cleanFileSet(testObject.getMongo_id(),Paths.piantaByIndex(0));
assertTrue(testObject.getPianteFineScavo().get(0).getActualContent().isEmpty());
}

23
geoportal-common/src/main/java/org/gcube/application/geoportal/common/faults/PathException.java Normal file
View File

@ -0,0 +1,23 @@
package org.gcube.application.geoportal.common.faults;
public class PathException extends Exception{
public PathException() {
}
public PathException(String message) {
super(message);
}
public PathException(String message, Throwable cause) {
super(message, cause);
}
public PathException(Throwable cause) {
super(cause);
}
public PathException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

22
geoportal-common/src/main/java/org/gcube/application/geoportal/common/faults/StorageException.java Normal file
View File

@ -0,0 +1,22 @@
package org.gcube.application.geoportal.common.faults;
public class StorageException extends Exception{
public StorageException() {
}
public StorageException(String message) {
super(message);
}
public StorageException(String message, Throwable cause) {
super(message, cause);
}
public StorageException(Throwable cause) {
super(cause);
}
public StorageException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

36
geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/StorageUtils.java
View File

@ -20,7 +20,8 @@ import lombok.extern.slf4j.Slf4j;
public class StorageUtils {
public static final IClient getClient(){
return new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME, ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient();
return new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME,
ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient();
}
private IClient client;
@ -42,26 +43,31 @@ public class StorageUtils {
else throw new FileNotFoundException("Unable to read "+filename);
}
public TempFile[] putOntoStorage(File... source) throws RemoteBackendException, FileNotFoundException{
TempFile[] toReturn=new TempFile[source.length];
for (int i = 0; i < source.length; i++) {
toReturn[i]=putOntoStorage(source[i],source[i].getName());
}
return toReturn;
}
public TempFile putOntoStorage(String sourcePath, String filename) throws RemoteBackendException, FileNotFoundException{
return putOntoStorage(new File(sourcePath),filename);
}
public static final boolean checkStorageId(String id){
return getClient().getHttpUrl().RFile(id)!=null;
}
public static final String getUrlById(String id){
IClient client=getClient();
log.debug("Id is "+id);
return client.getHttpUrl().RFile(id);
}
public static final void removeById(String id){
IClient client=getClient();
client.remove().RFile(id);
public void removeById(String id){
this.client.remove().RFile(id);
}
public String getURL(String id){
return this.client.getHttpsUrl().RFile(id);
}
public void exists(String id){
this.client.exist().RFile(id);
}
public static final String getUniqueString(){
return UUID.randomUUID().toString();
}

1
geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/WorkspaceManager.java
View File

@ -2,6 +2,7 @@ package org.gcube.application.geoportal.service.engine;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.geoportal.common.faults.PathException;
import org.gcube.application.geoportal.common.model.legacy.WorkspaceContent;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;

30
geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/mongo/ConcessioniMongoManager.java
View File

@ -6,12 +6,15 @@ import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.gcube.application.geoportal.common.faults.PathException;
import org.gcube.application.geoportal.common.faults.StorageException;
import org.gcube.application.geoportal.common.model.legacy.*;
import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport;
import org.gcube.application.geoportal.common.model.legacy.report.ValidationReport.ValidationStatus;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.rest.TempFile;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.geoportal.service.engine.ImplementationProvider;
import org.gcube.application.geoportal.service.engine.SDIManager;
import org.gcube.application.geoportal.service.engine.WorkspaceManager;
@ -22,10 +25,12 @@ import org.gcube.application.geoportal.service.engine.providers.StorageClientPro
import org.gcube.application.geoportal.service.model.internal.faults.*;
import org.gcube.application.geoportal.service.utils.Serialization;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.data.transfer.library.faults.RemoteServiceException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.ArrayList;
@ -261,6 +266,7 @@ public class ConcessioniMongoManager extends MongoManager{
log.info("Persisting {} files for path {} in concessione ",files.size(),destinationPath,id);
try{
Concessione c = getById(id);
StorageUtils storage=ImplementationProvider.get().getStorageProvider().getObject();
WorkspaceManager ws=new WorkspaceManager();
//Check Init Base folder
FolderContainer baseFolder=null;
@ -276,7 +282,7 @@ public class ConcessioniMongoManager extends MongoManager{
AssociatedContent section=c.getContentByPath(destinationPath);
log.debug("Found section {} for path {}",section,destinationPath);
store(section,files,ws,baseFolder);
store(section,files,ws,storage,baseFolder);
log.debug("Updating dafults for {} ",c);
c.setDefaults();
return asConcessione(replace(asDocument(c),collectionName));
@ -398,7 +404,7 @@ public class ConcessioniMongoManager extends MongoManager{
private static final void store(AssociatedContent content,List<TempFile> files, WorkspaceManager ws, FolderContainer base) throws Exception {
private static final void store(AssociatedContent content,List<TempFile> files, WorkspaceManager ws, StorageUtils storage, FolderContainer base) throws StorageHubException, StorageException, PathException {
FolderContainer sectionParent=null;
if(content instanceof RelazioneScavo)
@ -419,18 +425,26 @@ public class ConcessioniMongoManager extends MongoManager{
if(content instanceof LayerConcessione)
sectionParent = ws .createFolder(new FolderOptions(
content.getTitolo(),"Layer Concessione : "+content.getTitolo(),ws.getSubFolder(base,"layers")));
else throw new Exception("Invalid SDI Content "+content);
else throw new PathException("Invalid SDI Content "+content);
else if (content instanceof OtherContent )
sectionParent = ws .createFolder(new FolderOptions(
content.getTitolo(),"Relazione di scavo : "+content.getTitolo(),ws.getSubFolder(base,"other")));
else throw new Exception("Invalid Content "+content);
else throw new PathException("Invalid Content "+content);
content.setActualContent(new ArrayList<PersistedContent>());
StorageClientProvider storage=ImplementationProvider.get().getStorageProvider();
for(TempFile f : files) {
WorkspaceContent wsContent=ws.storeToWS(new FileOptions(f.getFilename(), storage.open(f.getId()), "Imported via GeoPortal", sectionParent));
log.debug("Registered "+wsContent+" for "+content);
content.getActualContent().add(wsContent);
try{
log.debug("Opening temp file {}",f);
String fileUrl=storage.getURL(f.getId());
log.debug("Got URL {} from ID {}",fileUrl,f.getId());
WorkspaceContent wsContent=ws.storeToWS(new FileOptions(f.getFilename(),new URL(fileUrl).openStream(),
"Imported via GeoPortal", sectionParent));
log.debug("Registered "+wsContent+" for "+content);
content.getActualContent().add(wsContent);
}catch(StorageHubException | IOException e){
throw new StorageException("Unable to store "+f,e);
}
}
content.setMongo_id(asString(new ObjectId()));
}

22
geoportal-service/src/main/java/org/gcube/application/geoportal/service/engine/providers/StorageClientProvider.java
View File

@ -2,6 +2,7 @@ package org.gcube.application.geoportal.service.engine.providers;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.geoportal.common.rest.InterfaceConstants;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
import org.gcube.application.geoportal.service.utils.ContextUtils;
import org.gcube.contentmanagement.blobstorage.service.IClient;
@ -18,7 +19,7 @@ import java.net.URL;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
@Slf4j
public class StorageClientProvider extends AbstractScopedMap<IClient> {
public class StorageClientProvider extends AbstractScopedMap<StorageUtils> {
@ -28,12 +29,13 @@ public class StorageClientProvider extends AbstractScopedMap<IClient> {
}
@Override
protected IClient retrieveObject() throws ConfigurationException {
return new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME, ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient();
protected StorageUtils retrieveObject() throws ConfigurationException {
return new StorageUtils();
// return new StorageClient(InterfaceConstants.SERVICE_CLASS, InterfaceConstants.SERVICE_NAME, ContextUtils.getCurrentCaller(), AccessType.SHARED, MemoryType.VOLATILE).getClient();
}
@Override
protected void dispose(IClient toDispose) {
protected void dispose(StorageUtils toDispose) {
try {
//TODO ASK
// toDispose.close();
@ -49,16 +51,4 @@ public class StorageClientProvider extends AbstractScopedMap<IClient> {
}
//wrapping methods
public InputStream open(String id) throws MalformedURLException, RemoteBackendException, IOException, ConfigurationException {
return new URL(getObject().getHttpsUrl().RFileById(id)).openConnection().getInputStream();
}
public String store(InputStream is) throws RemoteBackendException, ConfigurationException {
return getObject().put(true).LFile(is).RFile(Utils.getUniqueString());
}
}

20
geoportal-service/src/test/java/org/gcube/application/geoportal/service/StorageTests.java Normal file
View File

@ -0,0 +1,20 @@
package org.gcube.application.geoportal.service;
import org.gcube.application.geoportal.common.utils.StorageUtils;
import org.gcube.application.geoportal.service.engine.ImplementationProvider;
import org.gcube.application.geoportal.service.engine.providers.StorageClientProvider;
import org.gcube.application.geoportal.service.model.internal.faults.ConfigurationException;
import java.io.IOException;
public class StorageTests {
public static void main(String[] args) throws ConfigurationException, IOException {
TokenSetter.set("/gcube/devsec/devVRE");
StorageClientProvider storage= ImplementationProvider.get().getStorageProvider();
String id="614de23b647cef06aecdfb28";
System.out.println("FROM PROVIDER " + storage.getObject().getURL(id));
System.out.println("FROM CLASS" +new StorageUtils().getURL(id));
}
}