package org.gcube.application.cms.usecases; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione; import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.geoportal.client.utils.Serialization; import org.gcube.application.geoportal.common.model.configuration.Configuration; import org.gcube.application.geoportal.common.model.document.Project; import org.gcube.application.geoportal.common.model.document.access.Access; import org.gcube.application.geoportal.common.model.document.access.AccessPolicy; import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest; import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest; import org.gcube.application.geoportal.common.model.rest.TempFile; import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field; import org.gcube.application.geoportal.common.rest.Projects; import org.gcube.application.geoportal.common.utils.FileSets; import org.gcube.application.geoportal.common.utils.Files; import java.io.File; import java.nio.charset.Charset; import java.rmi.RemoteException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.stream.Collectors; import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPlugin.projects; @Slf4j public class ExportConcessioniAsProjects { @SneakyThrows private static final Concessione read(File jsonFile) { log.info("Reading "+jsonFile.getAbsolutePath()); String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset()); return Serialization.read(json,Concessione.class); } public static void main(String[] args) throws InterruptedException, RemoteException { // read from imported folder File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1666713419018"); // File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378"); String targetContext="/gcube/devsec/devVRE"; Projects client=projects("profiledConcessioni").build();; Integer totalCount = 0; Comparator comparator = new Comparator() { @Override public int compare(Concessione o1, Concessione o2) { return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto()); } }; // Title - > Time ordered list AtomicLong loadedCount = new AtomicLong(0); Map> relationshipMap = new HashMap<>(); for(File elementFolder:dir.listFiles()){ if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) { Concessione c =read(elementFolder); if(!relationshipMap.containsKey(c.getNome())) relationshipMap.put(c.getNome(),new ArrayList<>()); relationshipMap.get(c.getNome()).add(c); loadedCount.incrementAndGet(); } else for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})){ Concessione c =read(jsonFile); if(!relationshipMap.containsKey(c.getNome())) relationshipMap.put(c.getNome(),new ArrayList<>()); relationshipMap.get(c.getNome()).add(c); loadedCount.incrementAndGet(); } } System.out.println("Loaded "+loadedCount+" elements from "+dir.getAbsolutePath()); // order lists relationshipMap.forEach((s,l) ->{ log.info("Sorting {} ({} elements)",s,l.size()); Collections.sort(l,comparator); if(l.size()>1) { l.forEach(concessione -> log.info("{} : {} ",concessione.getMongo_id(),concessione.getDataInizioProgetto())); } }); TokenSetter.set(targetContext); // StorageUtils storage = new StorageUtils(); AtomicLong count = new AtomicLong(0); AtomicLong warnCount = new AtomicLong(0); AtomicLong errCount = new AtomicLong(0); ExecutorService service = Executors.newFixedThreadPool(3); long startProcess = System.currentTimeMillis(); relationshipMap.forEach((s,l)->{ String relationshipTarget=null; for (Concessione c : l) { String finalRelationshipTarget = relationshipTarget; CompletableFuture lastPublished = CompletableFuture.supplyAsync(new Supplier() { @Override public String get() { return publish(c, client, errCount, count, finalRelationshipTarget); } },service); try { relationshipTarget = lastPublished.get(); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ExecutionException e) { throw new RuntimeException(e); } } }); while (!service.awaitTermination(1, TimeUnit.MINUTES)) { log.info("Waiting .. completed {}, out of {} ",count.get(),loadedCount); if(loadedCount.get()==count.get()) service.shutdown(); } System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]"); Configuration conf = client.getConfiguration(); conf.getArchives().stream().forEach(archive -> { if (archive.getType().equals("DOCUMENT-STORE-COLLECTION")){ archive.forEach((s, o) ->{ System.out.println(s+"\t:\t"+o); }); } }); } private static String publish(Concessione c, Projects client, AtomicLong errCount, AtomicLong count, String relationshipTarget){ Project project = null; try { long startTime = System.currentTimeMillis(); FileSets.RequestBuilder req=null; log.info("Using {} {}",c.getNome(),c.getMongo_id()); //Copy core fields Document doc=new Document(); doc.put(ProfiledConcessione.NOME,c.getNome()); doc.put(ProfiledConcessione.INTRODUZIONE,c.getIntroduzione()); doc.put(ProfiledConcessione.DESCRIZIONE_CONTENUTO,c.getDescrizioneContenuto()); doc.put(ProfiledConcessione.AUTHORS,c.getAuthors()); doc.put(ProfiledConcessione.CONTRIBUTORE,c.getContributore()); doc.put(ProfiledConcessione.TITOLARI,c.getTitolari()); doc.put(ProfiledConcessione.RESPONSABILE,c.getResponsabile()); doc.put(ProfiledConcessione.EDITORS,c.getEditore()); doc.put(ProfiledConcessione.FONTI_FINANZIAMENTO,c.getFontiFinanziamento()); doc.put(ProfiledConcessione.SOGGETTO,c.getSoggetto()); doc.put(ProfiledConcessione.RISORSE_CORRELATE,c.getRisorseCorrelate()); doc.put(ProfiledConcessione.DATA_INZIO_PROGETTO,c.getDataInizioProgetto()); doc.put(ProfiledConcessione.DATA_FINE_PROGETTO,c.getDataFineProgetto()); doc.put(ProfiledConcessione.TITOLARE_LICENZA,c.getTitolareLicenza()); doc.put(ProfiledConcessione.TITOLARE_COPYRIGHT,c.getTitolareCopyright()); doc.put(ProfiledConcessione.PAROLE_CHIAVE_LIBERE,c.getParoleChiaveLibere()); doc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,c.getParoleChiaveICCD()); // CREATE Project project = client.createNew(doc); // set Access Access toSetAccess = new Access(); toSetAccess.setPolicy(AccessPolicy.valueOf(c.getPolicy().toString())); toSetAccess.setLicense(c.getLicenzaID()); project = client.setAccessPolicy(project.getId(),toSetAccess); // Relazione if(c.getRelazioneScavo()!=null) { RelazioneScavo rel=c.getRelazioneScavo(); Document relDoc= new Document(); relDoc.put(ProfiledConcessione.Sections.TITOLO,rel.getTitolo()); relDoc.put(ProfiledConcessione.Relazione.RESPONSABILI,rel.getResponsabili()); relDoc.put(ProfiledConcessione.SOGGETTO,rel.getSoggetto()); Document updated =project.getTheDocument(); updated.put(ProfiledConcessione.RELAZIONE_SCAVO,relDoc); project = client.updateDocument(project.getId(),updated); List content=rel.getActualContent(); if(content!=null && !content.isEmpty()) { req = FileSets.build("$." + ProfiledConcessione.RELAZIONE_SCAVO, "fileset", "$." + ProfiledConcessione.RELAZIONE_SCAVO + "." + Field.CHILDREN + "[?(@.fileset)]"); req.addAll(convertFiles(content)); req.setAccess(getAccess(rel)); project = client.registerFileSet(project.getId(),req.getTheRequest()); } // upload fileset } if(c.getAbstractRelazione()!=null) { AbstractRelazione abs=c.getAbstractRelazione(); Document absDoc= new Document(); absDoc.put(ProfiledConcessione.Sections.TITOLO,abs.getTitolo()); absDoc.put(ProfiledConcessione.AbstractRelazione.ABSTRACT_ENG,abs.getAbstractEng()); absDoc.put(ProfiledConcessione.AbstractRelazione.ABSTRACT_ITA,abs.getAbstractIta()); Document updated =project.getTheDocument(); updated.put(ProfiledConcessione.ABSTRACT_RELAZIONE,absDoc); project = client.updateDocument(project.getId(),updated); //TODO ask Fra about mapping // list persited content -> absIta, absEng // List content=abs.getActualContent(); // if(content!=null && !content.isEmpty()) { // req = FileSets.build("$." + ProfiledConcessione.RELAZIONE_SCAVO, "fileset", // "$." + ProfiledConcessione.RELAZIONE_SCAVO + "." + Field.CHILDREN // + "[?(@.fileset)]"); // content.forEach(uploader); // project = client.registerFileSet(project.getId(),req.getTheRequest()); // } } //Immagini rappresentative if(c.getImmaginiRappresentative()!=null){ List imgs=new ArrayList<>(); for (int i = 0; i < c.getImmaginiRappresentative().size(); i++) { UploadedImage img=c.getImmaginiRappresentative().get(i); Document imgDoc=new Document(); imgDoc.put(ProfiledConcessione.Sections.TITOLO,img.getTitolo()); imgDoc.put(ProfiledConcessione.Immagini.DIDASCALIA,img.getDidascalia()); imgDoc.put(ProfiledConcessione.Relazione.RESPONSABILI,img.getResponsabili()); imgDoc.put(ProfiledConcessione.SOGGETTO,img.getSoggetto()); imgs.add(imgDoc); } Document updated =project.getTheDocument(); updated.put(ProfiledConcessione.IMMAGINI_RAPPRESENTATIVE,imgs); project = client.updateDocument(project.getId(),updated); // FileSets for (int i = 0; i < c.getImmaginiRappresentative().size(); i++) { UploadedImage img=c.getImmaginiRappresentative().get(i); List content=img.getActualContent(); if(content!=null && !content.isEmpty()) { req = FileSets.build("$." + ProfiledConcessione.IMMAGINI_RAPPRESENTATIVE+"["+i+"]", "fileset", "$." + ProfiledConcessione.IMMAGINI_RAPPRESENTATIVE + "." + Field.CHILDREN + "[?(@.fileset)]"); req.addAll(convertFiles(content)); req.setAccess(getAccess(img)); project = client.registerFileSet(project.getId(),req.getTheRequest()); } } } // POS if(c.getPosizionamentoScavo()!=null) { LayerConcessione l=c.getPosizionamentoScavo(); Document lDoc= new Document(); lDoc.put(ProfiledConcessione.Sections.TITOLO,l.getTitolo()); lDoc.put(ProfiledConcessione.RESPONSABILE,l.getResponsabile()); lDoc.put(ProfiledConcessione.AUTHORS,l.getAuthors()); lDoc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,l.getParoleChiaveICCD()); lDoc.put(ProfiledConcessione.PAROLE_CHIAVE_LIBERE,l.getParoleChiaveLibere()); lDoc.put(ProfiledConcessione.Layers.TOPIC,l.getTopicCategory()); lDoc.put(ProfiledConcessione.Layers.SUB_TOPIC,l.getSubTopic()); lDoc.put(ProfiledConcessione.Layers.METODO_RACCOLTA,l.getMetodoRaccoltaDati()); lDoc.put(ProfiledConcessione.Layers.SCALA_ACQUISIZIONE,l.getScalaAcquisizione()); lDoc.put(ProfiledConcessione.Layers.VALUTAZIONE_QUALITA,l.getValutazioneQualita()); lDoc.put(ProfiledConcessione.Sections.ABSTRACT,l.getAbstractSection()); Document updated =project.getTheDocument(); updated.put(ProfiledConcessione.POSIZIONAMENTO_SCAVO,lDoc); project = client.updateDocument(project.getId(),updated); List content=l.getActualContent(); if(content!=null && !content.isEmpty()) { req = FileSets.build("$." + ProfiledConcessione.POSIZIONAMENTO_SCAVO, "fileset", "$." + ProfiledConcessione.POSIZIONAMENTO_SCAVO + "." + Field.CHILDREN + "[?(@.fileset)]"); req.addAll(convertFiles(content)); req.setAccess(getAccess(l)); project = client.registerFileSet(project.getId(),req.getTheRequest()); } // upload fileset } // Piante if(c.getPianteFineScavo()!=null){ List piante=new ArrayList<>(); for (int i = 0; i < c.getPianteFineScavo().size(); i++) { LayerConcessione l=c.getPianteFineScavo().get(i); Document lDoc= new Document(); lDoc.put(ProfiledConcessione.Sections.TITOLO,l.getTitolo()); lDoc.put(ProfiledConcessione.RESPONSABILE,l.getResponsabile()); lDoc.put(ProfiledConcessione.AUTHORS,l.getAuthors()); lDoc.put(ProfiledConcessione.PAREOLE_CHIAVE_ICCD,l.getParoleChiaveICCD()); lDoc.put(ProfiledConcessione.PAROLE_CHIAVE_LIBERE,l.getParoleChiaveLibere()); lDoc.put(ProfiledConcessione.Layers.TOPIC,l.getTopicCategory()); lDoc.put(ProfiledConcessione.Layers.SUB_TOPIC,l.getSubTopic()); lDoc.put(ProfiledConcessione.Layers.METODO_RACCOLTA,l.getMetodoRaccoltaDati()); lDoc.put(ProfiledConcessione.Layers.SCALA_ACQUISIZIONE,l.getScalaAcquisizione()); lDoc.put(ProfiledConcessione.Layers.VALUTAZIONE_QUALITA,l.getValutazioneQualita()); lDoc.put(ProfiledConcessione.Sections.ABSTRACT,l.getAbstractSection()); piante.add(lDoc); } Document updated =project.getTheDocument(); updated.put(ProfiledConcessione.PIANTE_FINE_SCAVO,piante); project = client.updateDocument(project.getId(),updated); // FileSets for (int i = 0; i < c.getPianteFineScavo().size(); i++) { LayerConcessione layerConcessione=c.getPianteFineScavo().get(i); List content=layerConcessione.getActualContent(); if(content!=null && !content.isEmpty()) { req = FileSets.build("$." + ProfiledConcessione.PIANTE_FINE_SCAVO+"["+i+"]", "fileset", "$." + ProfiledConcessione.PIANTE_FINE_SCAVO + "." + Field.CHILDREN + "[?(@.fileset)]"); req.addAll(convertFiles(content)); req.setAccess(getAccess(layerConcessione)); project = client.registerFileSet(project.getId(),req.getTheRequest()); } } } // Relationship if(relationshipTarget!=null){ CreateRelationshipRequest relReq = new CreateRelationshipRequest(); relReq.setProjectId(project.getId()); relReq.setRelationshipId("follows"); relReq.setTargetId(relationshipTarget); client.setRelation(relReq); } // Submit for review project = client.performStep(project.getId(),step()); System.out.println("Done "+c.getId()+" in "+(System.currentTimeMillis()-startTime)/1000+" sec"); } catch (Throwable throwable) { System.err.println(throwable); errCount.incrementAndGet(); } finally { count.incrementAndGet(); return project!=null? project.getId() : null; } } private static final List convertFiles(List content){ return content.stream(). filter(p -> p instanceof WorkspaceContent) .map(p -> new TempFile(null, ((WorkspaceContent) p).getLink(), ((WorkspaceContent) p).getName())) .collect(Collectors.toList()); } private static final Access getAccess(AssociatedContent content){ Access toReturn = new Access(); toReturn.setLicense(content.getLicenseID()); toReturn.setPolicy(AccessPolicy.valueOf(content.getPolicy().toString())); return toReturn; } private static final StepExecutionRequest step(){ StepExecutionRequest toReturn = new StepExecutionRequest(); toReturn.setStepID("SUBMIT-FOR-REVIEW"); return toReturn; } }