Compare commits

...

2 Commits

Author SHA1 Message Date
Fabio Sinibaldi 633cf8ee5c Docs 1 year ago
Fabio Sinibaldi a7001b8100 Utils 1 year ago

@ -2,14 +2,15 @@ package org.gcube.application.geoportal.common.utils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.xml.sax.InputSource;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.*;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.CopyOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.*;
@Slf4j
@ -112,4 +113,9 @@ public class Files {
if(is!=null) IOUtils.closeQuietly(is);
}
}
public static final void saveString(String content, Path dest) throws IOException {
java.nio.file.Files.copy(new ByteArrayInputStream(content.getBytes()),dest, StandardCopyOption.REPLACE_EXISTING);
}
}

@ -11,12 +11,11 @@ public class GCubeTest {
}
// testContext = "/pred4s/preprod/preVRE";
// InterfaceConstants.SERVICE_CLASS="Application";
// InterfaceConstants.SERVICE_NAME="GeoPortal";
testContext = "/gcube/devsec/devVRE";
// testContext = "/pred4s/preprod/preVRE";
testContext= "/gcube/devsec/devVRE";
System.out.println("TEST CONTEXT = "+testContext);
return testContext;

@ -7,19 +7,22 @@ Architecture
.. figure :: _static/imgs/architecture.png
:alt: CMS Suite overall concept
gCube CMS suite : General Architecture
gCube CMS suite : General Architecture
The conceptual architecture behind gCube CMS Suite is schematized in the figure above.
The conceptual architecture behind gCube CMS Suite is schematized in the figure above.
Here we describe the rationale behind the architecture :
Here we describe the rationale behind the architecture :
* Clients perform both CRUD and Query operations against the service both via using the client library or by directly interacting with the service REST interface.
* Clients perform both CRUD and Query operations against the service both via using the client library or by directly interacting with the service REST interface.
* They interact with the service producing the custom metadata document (stored in a Document Store DB) and registering related Filesets (stored in gCube StorageHub).
* Clients sends steps (pre configured in the collection UCD) requests to the service, which in turn delegate Lifecycle operations to configured implementations.
* Clients may also directly consume materialized projects in ad hoc engines (e.g. SDI or DBMS). For this purpose, the service exposes information needed by the client in order to properly consume the project collection.
* They interact with the service producing the custom metadata document (stored in a Document Store DB) and registering related Filesets (stored in gCube StorageHub).
* Clients sends steps (pre configured in the collection UCD) requests to the service, which in turn delegate Lifecycle operations to configured implementations.
In the figure below we further illustrate the various components involved in the typical execution of Projects' lifecycle
* Clients may also directly consume materialized projects in ad hoc engines (e.g. SDI or DBMS). For this purpose, the service exposes information needed by the client in order to properly consume the project collection.
.. figure :: _static/imgs/execution_schema.png
:alt: Step execution
@ -292,7 +295,20 @@ Following diagram shows the interfaces defined in the framework (package *org.gc
Plugin Development support
==========================
In order to facilitate the development of plugins, the following solutions may be of use to the developer :
* extension of an already provided LifecycleManager in order to override default behaviour (e.g. as per *Concessioni* UCD, see suite module concessioni-lifecycle)
* extension of *org.gcube.application.cms.plugins.implementations.AbstractPlugin* class in order to leverage on common behaiovur (e.g. configuration management, error handling)
* extension of *org.gcube.application.cms.caches.AbstractScopedMap* or *org.gcube.application.cms.caches.TimedMap* for easy implementation of caches (respectively by context and global)
* leverage on module *cms-test-commons* for exploitation of context management, basic plugin testing (both static and in a gCube context)
**************
Client Library
**************

@ -1,10 +1,12 @@
package org.gcube.application.cms.usecases;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.tests.TokenSetter;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.rest.Projects;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import java.rmi.RemoteException;
import java.util.ArrayList;
@ -21,11 +23,11 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl
public class ClearProjects {
public static void main(String[] args) throws RemoteException, InterruptedException {
String context="/gcube/devsec/devVRE";
// String context="/gcube/devsec/devVRE";
//String context="/pred4s/preprod/preVRE";
// String context="/pred4s/preprod/preVRE";
TokenSetter.set(context);
TokenSetter.set(GCubeTest.getContext());
Projects<Project> client=projects("profiledConcessioni").build();;
@ -42,8 +44,13 @@ public class ClearProjects {
AtomicLong found=new AtomicLong(0);
Iterator<Project> it=null;
it=client.query(new QueryRequest());
// it=manager.search("{\"nome\" : {$regex : \"Mock .*\"}, \"creationTime\" :{$gt : \"2021-10-18T13:58:53.326\"}}");
String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"Landro (Comune di Tambre, BL) \"},\"_theDocument.dataInizioProgetto\":{\"$eq\":\"2021-05-29T00:00:00\"}}\n" +
"}");
QueryRequest q = new QueryRequest();
q.setFilter(Document.parse(queryString));
it=client.query(q);
ExecutorService service = Executors.newFixedThreadPool(1);
@ -63,7 +70,7 @@ public class ClearProjects {
@Override
public void run() {
try{
TokenSetter.set(context);
TokenSetter.set(GCubeTest.getContext());
String currentId=c.getId();
if(currentId==null) {
System.out.println("ID IS NULL " + c);

@ -1,30 +1,37 @@
package org.gcube.application.cms.usecases;
import lombok.SneakyThrows;
import com.fasterxml.jackson.core.JsonProcessingException;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.cms.tests.TokenSetter;
import org.gcube.application.geoportal.client.utils.Serialization;
import org.gcube.application.cms.usecases.legacyConcessioni.ConcessioniManagementUtils;
import org.gcube.application.geoportal.common.model.configuration.Configuration;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
import org.gcube.application.geoportal.common.model.legacy.*;
import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest;
import org.gcube.application.geoportal.common.model.rest.TempFile;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
import org.gcube.application.geoportal.common.rest.Projects;
import org.gcube.application.geoportal.common.utils.FileSets;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import java.io.File;
import java.nio.charset.Charset;
import java.io.IOException;
import java.rmi.RemoteException;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -35,23 +42,27 @@ public class ExportConcessioniAsProjects {
@SneakyThrows
private static final Concessione read(File jsonFile) {
log.info("Reading "+jsonFile.getAbsolutePath());
String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
return Serialization.read(json,Concessione.class);
}
public static void main(String[] args) throws InterruptedException, RemoteException {
public static void main(String[] args) throws InterruptedException, IOException {
// read from imported folder
File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1666713419018");
File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348");
// select only cluster
// dir = new File(dir,"cluster_1");
File errorDir=new File(dir,"errors_"+ Instant.now().toString());
Boolean pushProjects=true;
// File dir= new File("test-data/test-packages/related_concessioni");
// File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378");
String targetContext="/gcube/devsec/devVRE";
//String targetContext="/gcube/devsec/devVRE";
// String targetContext="/pred4s/preprod/preVRE";
Projects<Project> client=projects("profiledConcessioni").build();;
@ -59,47 +70,19 @@ public class ExportConcessioniAsProjects {
Integer totalCount = 0;
Comparator<Concessione> comparator = new Comparator<Concessione>() {
@Override
public int compare(Concessione o1, Concessione o2) {
return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
}
};
// Title - > Time ordered list
AtomicLong loadedCount = new AtomicLong(0);
Map<String, ArrayList<Concessione>> relationshipMap = new HashMap<>();
for(File elementFolder:dir.listFiles()){
if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) {
Concessione c =read(elementFolder);
if(!relationshipMap.containsKey(c.getNome()))
relationshipMap.put(c.getNome(),new ArrayList<>());
relationshipMap.get(c.getNome()).add(c);
loadedCount.incrementAndGet();
}
else
for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})){
Concessione c =read(jsonFile);
if(!relationshipMap.containsKey(c.getNome()))
relationshipMap.put(c.getNome(),new ArrayList<>());
relationshipMap.get(c.getNome()).add(c);
loadedCount.incrementAndGet();
}
}
System.out.println("Loaded "+loadedCount+" elements from "+dir.getAbsolutePath());
// Title - > Time ordered list
ConcessioniManagementUtils.ImportFolder importFolder = ConcessioniManagementUtils.loadImportFolder(dir);
// order lists
relationshipMap.forEach((s,l) ->{
log.info("Sorting {} ({} elements)",s,l.size());
Collections.sort(l,comparator);
if(l.size()>1) {
l.forEach(concessione -> log.info("{} : {} ",concessione.getMongo_id(),concessione.getDataInizioProgetto()));
}
});
Files.saveString(importFolder.sort(),new File(dir,"relationships.csv").toPath());
if(!pushProjects){
System.out.println("PUSH PROJECTS FLAG IS FALSE.. CLOSING.");
System.exit(0);
}
TokenSetter.set(targetContext);
TokenSetter.set(GCubeTest.getContext());
// StorageUtils storage = new StorageUtils();
@ -107,38 +90,63 @@ public class ExportConcessioniAsProjects {
AtomicLong warnCount = new AtomicLong(0);
AtomicLong errCount = new AtomicLong(0);
ExecutorService service = Executors.newFixedThreadPool(1);
ExecutorService service = Executors.newFixedThreadPool(3);
long startProcess = System.currentTimeMillis();
relationshipMap.forEach((s,l)->{
String relationshipTarget=null;
for (Concessione c : l) {
String finalRelationshipTarget = relationshipTarget;
CompletableFuture<String> lastPublished = CompletableFuture.supplyAsync(new Supplier<String>() {
@Override
public String get() {
return publish(c, client, errCount, count, finalRelationshipTarget);
importFolder.getRelationshipMap().forEach((s,l)->{
AtomicReference<Boolean> isSetOk = new AtomicReference<>(true);
// Check if exists
Concessione c = l.get(0);
try {
List<Project> existing= getExisting(c,client);
if(existing.isEmpty()) {
log.info("Not Found");
}else {
existing.forEach(project -> {
if (!project.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
isSetOk.set(false);
});
if(!isSetOk.get()){
log.debug("Deleting error set for {}",c.getNome());
for (Project project : existing) {
client.deleteById(project.getId());
}
}
},service);
try {
relationshipTarget = lastPublished.get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}catch(NullPointerException e){}
catch (Throwable t){throw new RuntimeException("Unexpected Exception while checking for "+c.getNome());}
if(!isSetOk.get()) {
String relationshipTarget = null;
for (Concessione concessione : l) {
String finalRelationshipTarget = relationshipTarget;
CompletableFuture<String> lastPublished = CompletableFuture.supplyAsync(new Supplier<String>() {
@Override
public String get() {
return publish(concessione, client, errCount, count, finalRelationshipTarget, errorDir);
}
}, service);
try {
relationshipTarget = lastPublished.get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
}
});
while (!service.awaitTermination(1, TimeUnit.MINUTES)) {
log.info("Waiting .. completed {}, out of {} ",count.get(),loadedCount);
if(loadedCount.get()==count.get()) service.shutdown();
log.info("Waiting .. completed {}, out of {} ",count.get(),importFolder.getLoadedCount());
if(importFolder.getLoadedCount().get()==count.get()) service.shutdown();
}
System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]");
@ -160,7 +168,7 @@ public class ExportConcessioniAsProjects {
private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget){
private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget, File errorDir){
Project project = null;
try {
long startTime = System.currentTimeMillis();
@ -168,6 +176,8 @@ public class ExportConcessioniAsProjects {
log.info("Using {} {}",c.getNome(),c.getMongo_id());
//Copy core fields
Document doc=new Document();
@ -374,6 +384,17 @@ public class ExportConcessioniAsProjects {
} catch (Throwable throwable) {
System.err.println(throwable);
errCount.incrementAndGet();
String ser = null;
File destFile=null;
//if(relationshipTarget!=null) Files.saveString();
if(project!=null) {
ser = org.gcube.application.cms.serialization.Serialization.write(project);
destFile = new File(errorDir,"project_"+ project.getId());
}else{
ser = org.gcube.application.cms.serialization.Serialization.write(c);
destFile = new File(errorDir,"conc_"+ c.getMongo_id());
}
Files.saveString(ser, destFile.toPath());
} finally {
count.incrementAndGet();
return project!=null? project.getId() : null;
@ -401,4 +422,28 @@ public class ExportConcessioniAsProjects {
return toReturn;
}
private static List<Project> getExisting(Concessione c,Projects client) throws RemoteException, JsonProcessingException, NullPointerException {
try {
QueryRequest req = new QueryRequest();
String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"%1$s\"}}", c.getNome());
log.debug("Query String is {}", queryString);
req.setFilter(Document.parse(queryString));
AtomicInteger count = new AtomicInteger(0);
StringBuilder msg = new StringBuilder();
ArrayList<Project> toReturn = new ArrayList<>();
client.query(req).forEachRemaining(p -> {
Project proj = (Project) p;
count.incrementAndGet();
toReturn.add(proj);
msg.append(proj.getId() + ",");
});
log.debug("Found {} for {}", count.get(), c.getNome());
return toReturn;
}catch(Throwable t){
log.warn("ERROR while querying ",t);
throw t;
}
}
}

@ -18,9 +18,9 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl
public class ClearConcessioni {
public static void main(String[] args) throws Exception {
String context="/gcube/devsec/devVRE";
// String context="/gcube/devsec/devVRE";
//String context="/pred4s/preprod/preVRE";
String context="/pred4s/preprod/preVRE";
TokenSetter.set(context);

@ -0,0 +1,49 @@
package org.gcube.application.cms.usecases.legacyConcessioni;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.legacy.Concessione;
import org.gcube.application.geoportal.common.utils.Files;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
public class ClusterizeImportFolder {
public static void main(String[] args) throws IOException {
final File dir=new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348");
ConcessioniManagementUtils.ImportFolder folder=ConcessioniManagementUtils.loadImportFolder(dir);
folder.sort();
int clusterSize = 50;
int clusterNum=1;
File destDir =null;
for (Map.Entry<String, ArrayList<Concessione>> entry : folder.getRelationshipMap().entrySet()) {
String s = entry.getKey();
ArrayList<Concessione> l = entry.getValue();
if(destDir==null||destDir.list().length+l.size()>=clusterSize){
destDir = new File(dir, "cluster_" + clusterNum);
destDir.mkdirs();
clusterNum++;
}
File finalDestDir = destDir;
l.forEach(c -> {
try {
Files.saveString(Serialization.write(c),new File(finalDestDir,c.getMongo_id()+".json").toPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
System.out.println("Created "+clusterNum--+" sub folders");
}
}

@ -0,0 +1,85 @@
package org.gcube.application.cms.usecases.legacyConcessioni;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.geoportal.client.utils.Serialization;
import org.gcube.application.geoportal.common.model.legacy.Concessione;
import org.gcube.application.geoportal.common.utils.Files;
import java.io.File;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class ConcessioniManagementUtils {
@Data
public static class ImportFolder {
private Map<String, ArrayList<Concessione>> RelationshipMap=new HashMap<>();
private AtomicLong loadedCount=new AtomicLong(0l);
Comparator<Concessione> comparator = new Comparator<Concessione>() {
@Override
public int compare(Concessione o1, Concessione o2) {
return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
}
};
public String sort(){
StringBuilder reportBuilder=new StringBuilder();
// order lists
getRelationshipMap().forEach((s,l) ->{
log.info("Sorting {} ({} elements)",s,l.size());
Collections.sort(l,comparator);
l.forEach(concessione -> reportBuilder.append(concessione.getMongo_id()+ ",\"" +concessione.getNome()+"\","+ concessione.getDataInizioProgetto()+"\n"));
});
return reportBuilder.toString();
}
}
public static final ImportFolder loadImportFolder(File sourceDir){
ImportFolder toReturn = new ImportFolder();
for(File elementFolder:sourceDir.listFiles()){
if(elementFolder!=null&&!elementFolder.getName().equals("relationships.csv")) {
if (elementFolder.isFile() && elementFolder.getName().endsWith(".json")) {
Concessione c = read(elementFolder);
if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
toReturn.getRelationshipMap().get(c.getNome()).add(c);
toReturn.getLoadedCount().incrementAndGet();
} else
try{
for (File jsonFile : elementFolder.listFiles((dir1, name) -> {
return name!=null&&name.endsWith(".json");
})) {
Concessione c = read(jsonFile);
if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
toReturn.getRelationshipMap().get(c.getNome()).add(c);
toReturn.getLoadedCount().incrementAndGet();
}
}catch (Throwable t){
log.warn("Error while using "+elementFolder.getAbsolutePath(),t);
}
}
}
System.out.println("Loaded "+toReturn.getLoadedCount()+" elements from "+sourceDir.getAbsolutePath());
return toReturn;
}
@SneakyThrows
private static final Concessione read(File jsonFile) {
log.info("Reading "+jsonFile.getAbsolutePath());
String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
return Serialization.read(json,Concessione.class);
}
}

@ -18,9 +18,10 @@ public class DescribeWSFolder {
static StorageHubClient shc =null;
public static void main(String[] args) throws StorageHubException {
String context="/gcube/devsec/devVRE";
// String context="/gcube/devsec/devVRE";
String context="/pred4s/preprod/preVRE";
// String folderID="0518e4ad-0ef9-4cba-8d4f-8e315146acdd";
String folderID="a0d5ef8d-8e6a-4dd7-8539-5a2293e852cd";
String folderID="ded73475-f9b1-46c8-94f8-5b35c60667ce";
Boolean recursive = true;
TokenSetter.set(context);

Loading…
Cancel
Save