gCubeSystem
/
gcube-cms-suite
15
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Set relationships

This commit is contained in:
Fabio Sinibaldi 2022-10-26 18:01:09 +02:00
parent 6244694f0d
commit b8a0e8db4c
1 changed files with 89 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
use-cases/src/test/java/org/gcube/application/cms/usecases/ExportConcessioniAsProjects.java
View File

@ -11,6 +11,7 @@ import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.access.Access; import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy; import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.model.legacy.*;
import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest;
import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest; import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest;
import org.gcube.application.geoportal.common.model.rest.TempFile; import org.gcube.application.geoportal.common.model.rest.TempFile;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field; import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
@ -21,12 +22,10 @@ import org.gcube.application.geoportal.common.utils.Files;
import java.io.File; import java.io.File;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.rmi.RemoteException; import java.rmi.RemoteException;
import java.util.ArrayList; import java.util.*;
import java.util.List; import java.util.concurrent.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPlugin.projects; import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPlugin.projects;
@ -46,8 +45,8 @@ public class ExportConcessioniAsProjects {
public static void main(String[] args) throws InterruptedException, RemoteException { public static void main(String[] args) throws InterruptedException, RemoteException {
// read from imported folder // read from imported folder
File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1652276569045"); File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1666713419018");
// File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378"); // File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378");
String targetContext="/gcube/devsec/devVRE"; String targetContext="/gcube/devsec/devVRE";
@ -56,20 +55,50 @@ public class ExportConcessioniAsProjects {
ArrayList<Concessione> found=new ArrayList<>(); Integer totalCount = 0;
Comparator<Concessione> comparator = new Comparator<Concessione>() {
@Override
public int compare(Concessione o1, Concessione o2) {
return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
}
};
// Title - > Time ordered list
AtomicLong loadedCount = new AtomicLong(0);
Map<String, ArrayList<Concessione>> relationshipMap = new HashMap<>();
for(File elementFolder:dir.listFiles()){ for(File elementFolder:dir.listFiles()){
if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) {
found.add(read(elementFolder)); Concessione c =read(elementFolder);
if(!relationshipMap.containsKey(c.getNome()))
relationshipMap.put(c.getNome(),new ArrayList<>());
relationshipMap.get(c.getNome()).add(c);
loadedCount.incrementAndGet();
}
else else
for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})) for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})){
found.add(read(jsonFile)); Concessione c =read(jsonFile);
if(!relationshipMap.containsKey(c.getNome()))
relationshipMap.put(c.getNome(),new ArrayList<>());
relationshipMap.get(c.getNome()).add(c);
loadedCount.incrementAndGet();
}
} }
System.out.println("Loaded "+found.size()+" elements from "+dir.getAbsolutePath()); System.out.println("Loaded "+loadedCount+" elements from "+dir.getAbsolutePath());
// order lists
relationshipMap.forEach((s,l) ->{
log.info("Sorting {} ({} elements)",s,l.size());
Collections.sort(l,comparator);
if(l.size()>1) {
l.forEach(concessione -> log.info("{} : {} ",concessione.getMongo_id(),concessione.getDataInizioProgetto()));
}
});
TokenSetter.set(targetContext); TokenSetter.set(targetContext);
// StorageUtils storage = new StorageUtils(); // StorageUtils storage = new StorageUtils();
AtomicLong count = new AtomicLong(0); AtomicLong count = new AtomicLong(0);
AtomicLong warnCount = new AtomicLong(0); AtomicLong warnCount = new AtomicLong(0);
@ -79,20 +108,34 @@ public class ExportConcessioniAsProjects {
long startProcess = System.currentTimeMillis(); long startProcess = System.currentTimeMillis();
// TODO Relazioni
found.forEach(c -> { relationshipMap.forEach((s,l)->{
service.submit(new Runnable(){ String relationshipTarget=null;
@Override for (Concessione c : l) {
public void run() {
publish(c,client,errCount,count); String finalRelationshipTarget = relationshipTarget;
CompletableFuture<String> lastPublished = CompletableFuture.supplyAsync(new Supplier<String>() {
@Override
public String get() {
return publish(c, client, errCount, count, finalRelationshipTarget);
}
},service);
try {
relationshipTarget = lastPublished.get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
} }
} );
}
}); });
while (!service.awaitTermination(1, TimeUnit.MINUTES)) { while (!service.awaitTermination(1, TimeUnit.MINUTES)) {
log.info("Waiting .. completed {}, out of {} ",count.get(),found.size()); log.info("Waiting .. completed {}, out of {} ",count.get(),loadedCount);
if(found.size()==count.get()) service.shutdown(); if(loadedCount.get()==count.get()) service.shutdown();
} }
System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]"); System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]");
@ -102,7 +145,7 @@ public class ExportConcessioniAsProjects {
if (archive.getType().equals("DOCUMENT-STORE-COLLECTION")){ if (archive.getType().equals("DOCUMENT-STORE-COLLECTION")){
archive.forEach((s, o) ->{ archive.forEach((s, o) ->{
System.out.println(s+"\t:\t"+o); System.out.println(s+"\t:\t"+o);
}); });
} }
}); });
@ -110,7 +153,12 @@ public class ExportConcessioniAsProjects {
} }
private static void publish(Concessione c, Projects<Project> client, AtomicLong errCount,AtomicLong count){
private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget){
Project project = null;
try { try {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
FileSets.RequestBuilder req=null; FileSets.RequestBuilder req=null;
@ -139,7 +187,8 @@ public class ExportConcessioniAsProjects {
doc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,c.getParoleChiaveICCD()); doc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,c.getParoleChiaveICCD());
// CREATE Project // CREATE Project
Project project = client.createNew(doc);
project = client.createNew(doc);
// set Access // set Access
Access toSetAccess = new Access(); Access toSetAccess = new Access();
@ -302,15 +351,29 @@ public class ExportConcessioniAsProjects {
} }
} }
// Relationship
if(relationshipTarget!=null){
CreateRelationshipRequest relReq = new CreateRelationshipRequest();
relReq.setProjectId(project.getId());
relReq.setRelationshipId("follows");
relReq.setTargetId(relationshipTarget);
client.setRelation(relReq);
}
// Submit for review // Submit for review
project = client.performStep(project.getId(),step()); project = client.performStep(project.getId(),step());
System.out.println("Done "+c.getId()+" in "+(System.currentTimeMillis()-startTime)/1000+" sec"); System.out.println("Done "+c.getId()+" in "+(System.currentTimeMillis()-startTime)/1000+" sec");
} catch (Throwable throwable) { } catch (Throwable throwable) {
System.err.println(throwable); System.err.println(throwable);
errCount.incrementAndGet(); errCount.incrementAndGet();
} finally { } finally {
count.incrementAndGet(); count.incrementAndGet();
return project!=null? project.getId() : null;
} }
} }