402 lines
18 KiB
Java
402 lines
18 KiB
Java
package org.gcube.application.cms.usecases;
|
|
|
|
import lombok.SneakyThrows;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.bson.Document;
|
|
import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione;
|
|
import org.gcube.application.cms.tests.TokenSetter;
|
|
import org.gcube.application.geoportal.client.utils.Serialization;
|
|
import org.gcube.application.geoportal.common.model.configuration.Configuration;
|
|
import org.gcube.application.geoportal.common.model.document.Project;
|
|
import org.gcube.application.geoportal.common.model.document.access.Access;
|
|
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
|
|
import org.gcube.application.geoportal.common.model.legacy.*;
|
|
import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest;
|
|
import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest;
|
|
import org.gcube.application.geoportal.common.model.rest.TempFile;
|
|
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
|
|
import org.gcube.application.geoportal.common.rest.Projects;
|
|
import org.gcube.application.geoportal.common.utils.FileSets;
|
|
import org.gcube.application.geoportal.common.utils.Files;
|
|
|
|
import java.io.File;
|
|
import java.nio.charset.Charset;
|
|
import java.rmi.RemoteException;
|
|
import java.util.*;
|
|
import java.util.concurrent.*;
|
|
import java.util.concurrent.atomic.AtomicLong;
|
|
import java.util.function.Supplier;
|
|
import java.util.stream.Collectors;
|
|
|
|
import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPlugin.projects;
|
|
|
|
@Slf4j
|
|
public class ExportConcessioniAsProjects {
|
|
|
|
|
|
|
|
@SneakyThrows
|
|
private static final Concessione read(File jsonFile) {
|
|
log.info("Reading "+jsonFile.getAbsolutePath());
|
|
String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
|
|
return Serialization.read(json,Concessione.class);
|
|
}
|
|
|
|
|
|
public static void main(String[] args) throws InterruptedException, RemoteException {
|
|
// read from imported folder
|
|
File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1666713419018");
|
|
// File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378");
|
|
String targetContext="/gcube/devsec/devVRE";
|
|
|
|
|
|
Projects<Project> client=projects("profiledConcessioni").build();;
|
|
|
|
|
|
|
|
|
|
Integer totalCount = 0;
|
|
Comparator<Concessione> comparator = new Comparator<Concessione>() {
|
|
@Override
|
|
public int compare(Concessione o1, Concessione o2) {
|
|
return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
|
|
}
|
|
};
|
|
|
|
// Title - > Time ordered list
|
|
AtomicLong loadedCount = new AtomicLong(0);
|
|
Map<String, ArrayList<Concessione>> relationshipMap = new HashMap<>();
|
|
for(File elementFolder:dir.listFiles()){
|
|
if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) {
|
|
Concessione c =read(elementFolder);
|
|
if(!relationshipMap.containsKey(c.getNome()))
|
|
relationshipMap.put(c.getNome(),new ArrayList<>());
|
|
relationshipMap.get(c.getNome()).add(c);
|
|
loadedCount.incrementAndGet();
|
|
}
|
|
else
|
|
for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})){
|
|
Concessione c =read(jsonFile);
|
|
if(!relationshipMap.containsKey(c.getNome()))
|
|
relationshipMap.put(c.getNome(),new ArrayList<>());
|
|
relationshipMap.get(c.getNome()).add(c);
|
|
loadedCount.incrementAndGet();
|
|
}
|
|
}
|
|
|
|
System.out.println("Loaded "+loadedCount+" elements from "+dir.getAbsolutePath());
|
|
|
|
// order lists
|
|
relationshipMap.forEach((s,l) ->{
|
|
log.info("Sorting {} ({} elements)",s,l.size());
|
|
Collections.sort(l,comparator);
|
|
if(l.size()>1) {
|
|
l.forEach(concessione -> log.info("{} : {} ",concessione.getMongo_id(),concessione.getDataInizioProgetto()));
|
|
}
|
|
});
|
|
|
|
|
|
TokenSetter.set(targetContext);
|
|
|
|
// StorageUtils storage = new StorageUtils();
|
|
|
|
AtomicLong count = new AtomicLong(0);
|
|
AtomicLong warnCount = new AtomicLong(0);
|
|
AtomicLong errCount = new AtomicLong(0);
|
|
|
|
ExecutorService service = Executors.newFixedThreadPool(3);
|
|
|
|
long startProcess = System.currentTimeMillis();
|
|
|
|
|
|
relationshipMap.forEach((s,l)->{
|
|
String relationshipTarget=null;
|
|
for (Concessione c : l) {
|
|
|
|
String finalRelationshipTarget = relationshipTarget;
|
|
CompletableFuture<String> lastPublished = CompletableFuture.supplyAsync(new Supplier<String>() {
|
|
@Override
|
|
public String get() {
|
|
return publish(c, client, errCount, count, finalRelationshipTarget);
|
|
}
|
|
},service);
|
|
try {
|
|
relationshipTarget = lastPublished.get();
|
|
} catch (InterruptedException e) {
|
|
throw new RuntimeException(e);
|
|
} catch (ExecutionException e) {
|
|
throw new RuntimeException(e);
|
|
}
|
|
|
|
}
|
|
});
|
|
|
|
|
|
|
|
while (!service.awaitTermination(1, TimeUnit.MINUTES)) {
|
|
log.info("Waiting .. completed {}, out of {} ",count.get(),loadedCount);
|
|
if(loadedCount.get()==count.get()) service.shutdown();
|
|
}
|
|
|
|
System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]");
|
|
|
|
Configuration conf = client.getConfiguration();
|
|
conf.getArchives().stream().forEach(archive -> {
|
|
if (archive.getType().equals("DOCUMENT-STORE-COLLECTION")){
|
|
archive.forEach((s, o) ->{
|
|
|
|
System.out.println(s+"\t:\t"+o);
|
|
});
|
|
}
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget){
|
|
Project project = null;
|
|
try {
|
|
long startTime = System.currentTimeMillis();
|
|
FileSets.RequestBuilder req=null;
|
|
|
|
log.info("Using {} {}",c.getNome(),c.getMongo_id());
|
|
|
|
//Copy core fields
|
|
|
|
Document doc=new Document();
|
|
doc.put(ProfiledConcessione.NOME,c.getNome());
|
|
doc.put(ProfiledConcessione.INTRODUZIONE,c.getIntroduzione());
|
|
doc.put(ProfiledConcessione.DESCRIZIONE_CONTENUTO,c.getDescrizioneContenuto());
|
|
doc.put(ProfiledConcessione.AUTHORS,c.getAuthors());
|
|
doc.put(ProfiledConcessione.CONTRIBUTORE,c.getContributore());
|
|
doc.put(ProfiledConcessione.TITOLARI,c.getTitolari());
|
|
doc.put(ProfiledConcessione.RESPONSABILE,c.getResponsabile());
|
|
doc.put(ProfiledConcessione.EDITORS,c.getEditore());
|
|
doc.put(ProfiledConcessione.FONTI_FINANZIAMENTO,c.getFontiFinanziamento());
|
|
doc.put(ProfiledConcessione.SOGGETTO,c.getSoggetto());
|
|
doc.put(ProfiledConcessione.RISORSE_CORRELATE,c.getRisorseCorrelate());
|
|
doc.put(ProfiledConcessione.DATA_INZIO_PROGETTO,c.getDataInizioProgetto());
|
|
doc.put(ProfiledConcessione.DATA_FINE_PROGETTO,c.getDataFineProgetto());
|
|
doc.put(ProfiledConcessione.TITOLARE_LICENZA,c.getTitolareLicenza());
|
|
doc.put(ProfiledConcessione.TITOLARE_COPYRIGHT,c.getTitolareCopyright());
|
|
doc.put(ProfiledConcessione.PAROLE_CHIAVE_LIBERE,c.getParoleChiaveLibere());
|
|
doc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,c.getParoleChiaveICCD());
|
|
|
|
// CREATE Project
|
|
|
|
project = client.createNew(doc);
|
|
|
|
// set Access
|
|
Access toSetAccess = new Access();
|
|
toSetAccess.setPolicy(AccessPolicy.valueOf(c.getPolicy().toString()));
|
|
toSetAccess.setLicense(c.getLicenzaID());
|
|
project = client.setAccessPolicy(project.getId(),toSetAccess);
|
|
|
|
|
|
// Relazione
|
|
if(c.getRelazioneScavo()!=null) {
|
|
RelazioneScavo rel=c.getRelazioneScavo();
|
|
Document relDoc= new Document();
|
|
relDoc.put(ProfiledConcessione.Sections.TITOLO,rel.getTitolo());
|
|
relDoc.put(ProfiledConcessione.Relazione.RESPONSABILI,rel.getResponsabili());
|
|
relDoc.put(ProfiledConcessione.SOGGETTO,rel.getSoggetto());
|
|
|
|
Document updated =project.getTheDocument();
|
|
updated.put(ProfiledConcessione.RELAZIONE_SCAVO,relDoc);
|
|
project = client.updateDocument(project.getId(),updated);
|
|
|
|
List<PersistedContent> content=rel.getActualContent();
|
|
if(content!=null && !content.isEmpty()) {
|
|
req = FileSets.build("$." + ProfiledConcessione.RELAZIONE_SCAVO, "fileset",
|
|
"$." + ProfiledConcessione.RELAZIONE_SCAVO + "." + Field.CHILDREN
|
|
+ "[?(@.fileset)]");
|
|
req.addAll(convertFiles(content));
|
|
req.setAccess(getAccess(rel));
|
|
project = client.registerFileSet(project.getId(),req.getTheRequest());
|
|
}
|
|
// upload fileset
|
|
}
|
|
|
|
|
|
if(c.getAbstractRelazione()!=null) {
|
|
AbstractRelazione abs=c.getAbstractRelazione();
|
|
Document absDoc= new Document();
|
|
absDoc.put(ProfiledConcessione.Sections.TITOLO,abs.getTitolo());
|
|
absDoc.put(ProfiledConcessione.AbstractRelazione.ABSTRACT_ENG,abs.getAbstractEng());
|
|
absDoc.put(ProfiledConcessione.AbstractRelazione.ABSTRACT_ITA,abs.getAbstractIta());
|
|
|
|
|
|
Document updated =project.getTheDocument();
|
|
updated.put(ProfiledConcessione.ABSTRACT_RELAZIONE,absDoc);
|
|
project = client.updateDocument(project.getId(),updated);
|
|
|
|
//TODO ask Fra about mapping
|
|
// list persited content -> absIta, absEng
|
|
|
|
// List<PersistedContent> content=abs.getActualContent();
|
|
// if(content!=null && !content.isEmpty()) {
|
|
// req = FileSets.build("$." + ProfiledConcessione.RELAZIONE_SCAVO, "fileset",
|
|
// "$." + ProfiledConcessione.RELAZIONE_SCAVO + "." + Field.CHILDREN
|
|
// + "[?(@.fileset)]");
|
|
// content.forEach(uploader);
|
|
// project = client.registerFileSet(project.getId(),req.getTheRequest());
|
|
// }
|
|
|
|
}
|
|
|
|
//Immagini rappresentative
|
|
if(c.getImmaginiRappresentative()!=null){
|
|
List<Document> imgs=new ArrayList<>();
|
|
for (int i = 0; i < c.getImmaginiRappresentative().size(); i++) {
|
|
UploadedImage img=c.getImmaginiRappresentative().get(i);
|
|
Document imgDoc=new Document();
|
|
imgDoc.put(ProfiledConcessione.Sections.TITOLO,img.getTitolo());
|
|
imgDoc.put(ProfiledConcessione.Immagini.DIDASCALIA,img.getDidascalia());
|
|
imgDoc.put(ProfiledConcessione.Relazione.RESPONSABILI,img.getResponsabili());
|
|
imgDoc.put(ProfiledConcessione.SOGGETTO,img.getSoggetto());
|
|
imgs.add(imgDoc);
|
|
}
|
|
|
|
Document updated =project.getTheDocument();
|
|
updated.put(ProfiledConcessione.IMMAGINI_RAPPRESENTATIVE,imgs);
|
|
project = client.updateDocument(project.getId(),updated);
|
|
|
|
// FileSets
|
|
for (int i = 0; i < c.getImmaginiRappresentative().size(); i++) {
|
|
UploadedImage img=c.getImmaginiRappresentative().get(i);
|
|
List<PersistedContent> content=img.getActualContent();
|
|
if(content!=null && !content.isEmpty()) {
|
|
req = FileSets.build("$." + ProfiledConcessione.IMMAGINI_RAPPRESENTATIVE+"["+i+"]", "fileset",
|
|
"$." + ProfiledConcessione.IMMAGINI_RAPPRESENTATIVE + "." + Field.CHILDREN
|
|
+ "[?(@.fileset)]");
|
|
req.addAll(convertFiles(content));
|
|
req.setAccess(getAccess(img));
|
|
project = client.registerFileSet(project.getId(),req.getTheRequest());
|
|
}
|
|
}
|
|
}
|
|
|
|
// POS
|
|
if(c.getPosizionamentoScavo()!=null) {
|
|
LayerConcessione l=c.getPosizionamentoScavo();
|
|
Document lDoc= new Document();
|
|
lDoc.put(ProfiledConcessione.Sections.TITOLO,l.getTitolo());
|
|
lDoc.put(ProfiledConcessione.RESPONSABILE,l.getResponsabile());
|
|
lDoc.put(ProfiledConcessione.AUTHORS,l.getAuthors());
|
|
lDoc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,l.getParoleChiaveICCD());
|
|
lDoc.put(ProfiledConcessione.PAROLE_CHIAVE_LIBERE,l.getParoleChiaveLibere());
|
|
lDoc.put(ProfiledConcessione.Layers.TOPIC,l.getTopicCategory());
|
|
lDoc.put(ProfiledConcessione.Layers.SUB_TOPIC,l.getSubTopic());
|
|
lDoc.put(ProfiledConcessione.Layers.METODO_RACCOLTA,l.getMetodoRaccoltaDati());
|
|
lDoc.put(ProfiledConcessione.Layers.SCALA_ACQUISIZIONE,l.getScalaAcquisizione());
|
|
lDoc.put(ProfiledConcessione.Layers.VALUTAZIONE_QUALITA,l.getValutazioneQualita());
|
|
lDoc.put(ProfiledConcessione.Sections.ABSTRACT,l.getAbstractSection());
|
|
|
|
Document updated =project.getTheDocument();
|
|
updated.put(ProfiledConcessione.POSIZIONAMENTO_SCAVO,lDoc);
|
|
project = client.updateDocument(project.getId(),updated);
|
|
|
|
List<PersistedContent> content=l.getActualContent();
|
|
if(content!=null && !content.isEmpty()) {
|
|
req = FileSets.build("$." + ProfiledConcessione.POSIZIONAMENTO_SCAVO, "fileset",
|
|
"$." + ProfiledConcessione.POSIZIONAMENTO_SCAVO + "." + Field.CHILDREN
|
|
+ "[?(@.fileset)]");
|
|
req.addAll(convertFiles(content));
|
|
req.setAccess(getAccess(l));
|
|
project = client.registerFileSet(project.getId(),req.getTheRequest());
|
|
}
|
|
// upload fileset
|
|
}
|
|
|
|
// Piante
|
|
if(c.getPianteFineScavo()!=null){
|
|
List<Document> piante=new ArrayList<>();
|
|
for (int i = 0; i < c.getPianteFineScavo().size(); i++) {
|
|
LayerConcessione l=c.getPianteFineScavo().get(i);
|
|
Document lDoc= new Document();
|
|
lDoc.put(ProfiledConcessione.Sections.TITOLO,l.getTitolo());
|
|
lDoc.put(ProfiledConcessione.RESPONSABILE,l.getResponsabile());
|
|
lDoc.put(ProfiledConcessione.AUTHORS,l.getAuthors());
|
|
lDoc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,l.getParoleChiaveICCD());
|
|
lDoc.put(ProfiledConcessione.PAROLE_CHIAVE_LIBERE,l.getParoleChiaveLibere());
|
|
lDoc.put(ProfiledConcessione.Layers.TOPIC,l.getTopicCategory());
|
|
lDoc.put(ProfiledConcessione.Layers.SUB_TOPIC,l.getSubTopic());
|
|
lDoc.put(ProfiledConcessione.Layers.METODO_RACCOLTA,l.getMetodoRaccoltaDati());
|
|
lDoc.put(ProfiledConcessione.Layers.SCALA_ACQUISIZIONE,l.getScalaAcquisizione());
|
|
lDoc.put(ProfiledConcessione.Layers.VALUTAZIONE_QUALITA,l.getValutazioneQualita());
|
|
lDoc.put(ProfiledConcessione.Sections.ABSTRACT,l.getAbstractSection());
|
|
piante.add(lDoc);
|
|
}
|
|
|
|
Document updated =project.getTheDocument();
|
|
updated.put(ProfiledConcessione.PIANTE_FINE_SCAVO,piante);
|
|
project = client.updateDocument(project.getId(),updated);
|
|
|
|
// FileSets
|
|
for (int i = 0; i < c.getPianteFineScavo().size(); i++) {
|
|
LayerConcessione layerConcessione=c.getPianteFineScavo().get(i);
|
|
List<PersistedContent> content=layerConcessione.getActualContent();
|
|
if(content!=null && !content.isEmpty()) {
|
|
req = FileSets.build("$." + ProfiledConcessione.PIANTE_FINE_SCAVO+"["+i+"]", "fileset",
|
|
"$." + ProfiledConcessione.PIANTE_FINE_SCAVO + "." + Field.CHILDREN
|
|
+ "[?(@.fileset)]");
|
|
req.addAll(convertFiles(content));
|
|
req.setAccess(getAccess(layerConcessione));
|
|
project = client.registerFileSet(project.getId(),req.getTheRequest());
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// Relationship
|
|
if(relationshipTarget!=null){
|
|
CreateRelationshipRequest relReq = new CreateRelationshipRequest();
|
|
relReq.setProjectId(project.getId());
|
|
relReq.setRelationshipId("follows");
|
|
relReq.setTargetId(relationshipTarget);
|
|
client.setRelation(relReq);
|
|
}
|
|
|
|
|
|
// Submit for review
|
|
project = client.performStep(project.getId(),step());
|
|
|
|
System.out.println("Done "+c.getId()+" in "+(System.currentTimeMillis()-startTime)/1000+" sec");
|
|
|
|
|
|
} catch (Throwable throwable) {
|
|
System.err.println(throwable);
|
|
errCount.incrementAndGet();
|
|
} finally {
|
|
count.incrementAndGet();
|
|
return project!=null? project.getId() : null;
|
|
}
|
|
}
|
|
|
|
|
|
private static final List<TempFile> convertFiles(List<PersistedContent> content){
|
|
return content.stream().
|
|
filter(p -> p instanceof WorkspaceContent)
|
|
.map(p -> new TempFile(null, ((WorkspaceContent) p).getLink(), ((WorkspaceContent) p).getName()))
|
|
.collect(Collectors.toList());
|
|
}
|
|
|
|
private static final Access getAccess(AssociatedContent content){
|
|
Access toReturn = new Access();
|
|
toReturn.setLicense(content.getLicenseID());
|
|
toReturn.setPolicy(AccessPolicy.valueOf(content.getPolicy().toString()));
|
|
return toReturn;
|
|
}
|
|
|
|
private static final StepExecutionRequest step(){
|
|
StepExecutionRequest toReturn = new StepExecutionRequest();
|
|
toReturn.setStepID("SUBMIT-FOR-REVIEW");
|
|
return toReturn;
|
|
}
|
|
|
|
}
|