Utils

geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java

@@ -2,14 +2,15 @@ package org.gcube.application.geoportal.common.utils;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
+import org.xml.sax.InputSource;
 
-import java.io.File;
+import java.io.*;
-import java.io.IOException;
-import java.io.InputStream;
 import java.net.URL;
 import java.nio.charset.Charset;
+import java.nio.file.CopyOption;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.*;
 
 @Slf4j
@@ -112,4 +113,9 @@ public class Files {
 			if(is!=null) IOUtils.closeQuietly(is);
 		}
 	}
+
+
+	public static final void saveString(String content, Path dest) throws IOException {
+		java.nio.file.Files.copy(new ByteArrayInputStream(content.getBytes()),dest, StandardCopyOption.REPLACE_EXISTING);
+	}
 }

geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/tests/GCubeTest.java

@@ -11,12 +11,11 @@ public class GCubeTest {
             }
 
 
-//          testContext = "/pred4s/preprod/preVRE";
 //            InterfaceConstants.SERVICE_CLASS="Application";
 //            InterfaceConstants.SERVICE_NAME="GeoPortal";
 
-            testContext = "/gcube/devsec/devVRE";
+           // testContext = "/pred4s/preprod/preVRE";
-
+          testContext= "/gcube/devsec/devVRE";
 
             System.out.println("TEST CONTEXT = "+testContext);
             return testContext;

use-cases/src/test/java/org/gcube/application/cms/usecases/ClearProjects.java

@@ -1,10 +1,12 @@
 package org.gcube.application.cms.usecases;
 
 import lombok.extern.slf4j.Slf4j;
+import org.bson.Document;
 import org.gcube.application.cms.tests.TokenSetter;
 import org.gcube.application.geoportal.common.model.document.Project;
 import org.gcube.application.geoportal.common.model.rest.QueryRequest;
 import org.gcube.application.geoportal.common.rest.Projects;
+import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
 
 import java.rmi.RemoteException;
 import java.util.ArrayList;
@@ -21,11 +23,11 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl
 public class ClearProjects {
 
     public static void main(String[] args) throws RemoteException, InterruptedException {
-        String context="/gcube/devsec/devVRE";
+       // String context="/gcube/devsec/devVRE";
 
-        //String context="/pred4s/preprod/preVRE";
+      //  String context="/pred4s/preprod/preVRE";
 
-        TokenSetter.set(context);
+        TokenSetter.set(GCubeTest.getContext());
 
         Projects<Project> client=projects("profiledConcessioni").build();;
 
@@ -42,8 +44,13 @@ public class ClearProjects {
         AtomicLong found=new AtomicLong(0);
 
         Iterator<Project> it=null;
-        it=client.query(new QueryRequest());
+
-//        it=manager.search("{\"nome\" : {$regex : \"Mock .*\"}, \"creationTime\" :{$gt : \"2021-10-18T13:58:53.326\"}}");
+        String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"Landro (Comune di Tambre, BL) \"},\"_theDocument.dataInizioProgetto\":{\"$eq\":\"2021-05-29T00:00:00\"}}\n" +
+                "}");
+        QueryRequest q = new QueryRequest();
+        q.setFilter(Document.parse(queryString));
+        it=client.query(q);
+
 
         ExecutorService service = Executors.newFixedThreadPool(1);
 
@@ -63,7 +70,7 @@ public class ClearProjects {
                 @Override
                 public void run() {
                     try{
-                        TokenSetter.set(context);
+                        TokenSetter.set(GCubeTest.getContext());
                         String currentId=c.getId();
                         if(currentId==null) {
                             System.out.println("ID IS NULL " + c);

use-cases/src/test/java/org/gcube/application/cms/usecases/ExportConcessioniAsProjects.java

@@ -1,30 +1,37 @@
 package org.gcube.application.cms.usecases;
 
-import lombok.SneakyThrows;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import lombok.extern.slf4j.Slf4j;
 import org.bson.Document;
 import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione;
+import org.gcube.application.cms.serialization.Serialization;
 import org.gcube.application.cms.tests.TokenSetter;
-import org.gcube.application.geoportal.client.utils.Serialization;
+import org.gcube.application.cms.usecases.legacyConcessioni.ConcessioniManagementUtils;
 import org.gcube.application.geoportal.common.model.configuration.Configuration;
 import org.gcube.application.geoportal.common.model.document.Project;
 import org.gcube.application.geoportal.common.model.document.access.Access;
 import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
+import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
 import org.gcube.application.geoportal.common.model.legacy.*;
 import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest;
+import org.gcube.application.geoportal.common.model.rest.QueryRequest;
 import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest;
 import org.gcube.application.geoportal.common.model.rest.TempFile;
 import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
 import org.gcube.application.geoportal.common.rest.Projects;
 import org.gcube.application.geoportal.common.utils.FileSets;
 import org.gcube.application.geoportal.common.utils.Files;
+import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
 
 import java.io.File;
-import java.nio.charset.Charset;
+import java.io.IOException;
 import java.rmi.RemoteException;
+import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -35,23 +42,27 @@ public class ExportConcessioniAsProjects {
 
 
 
-    @SneakyThrows
-    private static final Concessione read(File jsonFile) {
-        log.info("Reading "+jsonFile.getAbsolutePath());
-        String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
-        return Serialization.read(json,Concessione.class);
-    }
 
 
-    public static void main(String[] args) throws InterruptedException, RemoteException {
+
+    public static void main(String[] args) throws InterruptedException, IOException {
         // read from imported folder
-         File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1666713419018");
+         File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348");
+
+         // select only cluster
+//        dir = new File(dir,"cluster_1");
+
+
+
+         File errorDir=new File(dir,"errors_"+ Instant.now().toString());
+
+         Boolean pushProjects=true;
 
        // File dir= new File("test-data/test-packages/related_concessioni");
 
         // File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378");
-        String targetContext="/gcube/devsec/devVRE";
+        //String targetContext="/gcube/devsec/devVRE";
-
+       // String targetContext="/pred4s/preprod/preVRE";
 
         Projects<Project> client=projects("profiledConcessioni").build();;
 
@@ -59,47 +70,19 @@ public class ExportConcessioniAsProjects {
 
 
         Integer totalCount = 0;
-        Comparator<Concessione> comparator = new Comparator<Concessione>() {
+
-            @Override
-            public int compare(Concessione o1, Concessione o2) {
-                return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
-            }
-        };
 
         // Title - > Time ordered list
-        AtomicLong loadedCount = new AtomicLong(0);
+        ConcessioniManagementUtils.ImportFolder importFolder = ConcessioniManagementUtils.loadImportFolder(dir);
-        Map<String, ArrayList<Concessione>> relationshipMap = new HashMap<>();
+
-        for(File elementFolder:dir.listFiles()){
+        Files.saveString(importFolder.sort(),new File(dir,"relationships.csv").toPath());
-            if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) {
+
-                Concessione c =read(elementFolder);
+        if(!pushProjects){
-                if(!relationshipMap.containsKey(c.getNome()))
+            System.out.println("PUSH PROJECTS FLAG IS FALSE.. CLOSING.");
-                    relationshipMap.put(c.getNome(),new ArrayList<>());
+            System.exit(0);
-                relationshipMap.get(c.getNome()).add(c);
-                loadedCount.incrementAndGet();
-            }
-            else
-                for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})){
-                    Concessione c =read(jsonFile);
-                    if(!relationshipMap.containsKey(c.getNome()))
-                        relationshipMap.put(c.getNome(),new ArrayList<>());
-                    relationshipMap.get(c.getNome()).add(c);
-                    loadedCount.incrementAndGet();
-                }
         }
 
-        System.out.println("Loaded "+loadedCount+" elements from "+dir.getAbsolutePath());
+        TokenSetter.set(GCubeTest.getContext());
-
-        // order lists
-        relationshipMap.forEach((s,l) ->{
-            log.info("Sorting {} ({} elements)",s,l.size());
-            Collections.sort(l,comparator);
-            if(l.size()>1) {
-                l.forEach(concessione -> log.info("{} : {} ",concessione.getMongo_id(),concessione.getDataInizioProgetto()));
-            }
-        });
-
-
-        TokenSetter.set(targetContext);
 
         // StorageUtils storage = new StorageUtils();
 
@@ -107,38 +90,63 @@ public class ExportConcessioniAsProjects {
         AtomicLong warnCount = new AtomicLong(0);
         AtomicLong errCount = new AtomicLong(0);
 
-        ExecutorService service = Executors.newFixedThreadPool(1);
+        ExecutorService service = Executors.newFixedThreadPool(3);
 
         long startProcess = System.currentTimeMillis();
 
+        importFolder.getRelationshipMap().forEach((s,l)->{
 
-        relationshipMap.forEach((s,l)->{
+            AtomicReference<Boolean> isSetOk = new AtomicReference<>(true);
-            String relationshipTarget=null;
+            // Check if exists
-            for (Concessione c : l) {
+            Concessione c = l.get(0);
-
+            try {
-                String finalRelationshipTarget = relationshipTarget;
+                List<Project> existing= getExisting(c,client);
-                CompletableFuture<String> lastPublished = CompletableFuture.supplyAsync(new Supplier<String>() {
+                if(existing.isEmpty()) {
-                    @Override
+                    log.info("Not Found");
-                    public String get() {
+                }else {
-                        return  publish(c, client, errCount, count, finalRelationshipTarget);
+                    existing.forEach(project -> {
+                        if (!project.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
+                            isSetOk.set(false);
+                    });
+                    if(!isSetOk.get()){
+                        log.debug("Deleting error set for {}",c.getNome());
+                        for (Project project : existing) {
+                            client.deleteById(project.getId());
+                        }
                     }
-                },service);
-                try {
-                    relationshipTarget = lastPublished.get();
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
-                } catch (ExecutionException e) {
-                    throw new RuntimeException(e);
                 }
+            }catch(NullPointerException e){}
+            catch (Throwable t){throw new RuntimeException("Unexpected Exception while checking for "+c.getNome());}
 
+
+            if(!isSetOk.get()) {
+                String relationshipTarget = null;
+                for (Concessione concessione : l) {
+
+                    String finalRelationshipTarget = relationshipTarget;
+                    CompletableFuture<String> lastPublished = CompletableFuture.supplyAsync(new Supplier<String>() {
+                        @Override
+                        public String get() {
+                            return publish(concessione, client, errCount, count, finalRelationshipTarget, errorDir);
+                        }
+                    }, service);
+                    try {
+                        relationshipTarget = lastPublished.get();
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    } catch (ExecutionException e) {
+                        throw new RuntimeException(e);
+                    }
+
+                }
             }
         });
 
 
 
         while (!service.awaitTermination(1, TimeUnit.MINUTES)) {
-            log.info("Waiting .. completed {}, out of {} ",count.get(),loadedCount);
+            log.info("Waiting .. completed {}, out of {} ",count.get(),importFolder.getLoadedCount());
-            if(loadedCount.get()==count.get()) service.shutdown();
+            if(importFolder.getLoadedCount().get()==count.get()) service.shutdown();
         }
 
         System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]");
@@ -160,7 +168,7 @@ public class ExportConcessioniAsProjects {
 
 
 
-    private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget){
+    private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget, File errorDir){
         Project project = null;
         try {
             long startTime = System.currentTimeMillis();
@@ -168,6 +176,8 @@ public class ExportConcessioniAsProjects {
 
             log.info("Using {} {}",c.getNome(),c.getMongo_id());
 
+
+
             //Copy core fields
 
             Document doc=new Document();
@@ -374,6 +384,17 @@ public class ExportConcessioniAsProjects {
         } catch (Throwable throwable) {
             System.err.println(throwable);
             errCount.incrementAndGet();
+            String ser = null;
+            File destFile=null;
+            //if(relationshipTarget!=null) Files.saveString();
+            if(project!=null) {
+                ser = org.gcube.application.cms.serialization.Serialization.write(project);
+                destFile = new File(errorDir,"project_"+ project.getId());
+            }else{
+                ser = org.gcube.application.cms.serialization.Serialization.write(c);
+                destFile = new File(errorDir,"conc_"+ c.getMongo_id());
+            }
+            Files.saveString(ser, destFile.toPath());
         } finally {
             count.incrementAndGet();
             return project!=null? project.getId() : null;
@@ -401,4 +422,28 @@ public class ExportConcessioniAsProjects {
         return toReturn;
     }
 
+
+    private static List<Project> getExisting(Concessione c,Projects client) throws RemoteException, JsonProcessingException, NullPointerException {
+        try {
+            QueryRequest req = new QueryRequest();
+            String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"%1$s\"}}", c.getNome());
+            log.debug("Query String is {}", queryString);
+            req.setFilter(Document.parse(queryString));
+            AtomicInteger count = new AtomicInteger(0);
+            StringBuilder msg = new StringBuilder();
+            ArrayList<Project> toReturn = new ArrayList<>();
+
+            client.query(req).forEachRemaining(p -> {
+                Project proj = (Project) p;
+                count.incrementAndGet();
+                toReturn.add(proj);
+                msg.append(proj.getId() + ",");
+            });
+            log.debug("Found {} for {}", count.get(), c.getNome());
+            return toReturn;
+        }catch(Throwable t){
+            log.warn("ERROR while querying ",t);
+            throw t;
+        }
+    }
 }

use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClearConcessioni.java

@@ -18,9 +18,9 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl
 public class ClearConcessioni {
 
     public static void main(String[] args) throws Exception {
-        String context="/gcube/devsec/devVRE";
+      //  String context="/gcube/devsec/devVRE";
 
-        //String context="/pred4s/preprod/preVRE";
+        String context="/pred4s/preprod/preVRE";
 
         TokenSetter.set(context);
 

use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClusterizeImportFolder.java

@@ -0,0 +1,49 @@
+package org.gcube.application.cms.usecases.legacyConcessioni;
+
+import org.gcube.application.cms.serialization.Serialization;
+import org.gcube.application.geoportal.common.model.legacy.Concessione;
+import org.gcube.application.geoportal.common.utils.Files;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
+public class ClusterizeImportFolder {
+
+
+    public static void main(String[] args) throws IOException {
+        final File dir=new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348");
+
+
+        ConcessioniManagementUtils.ImportFolder folder=ConcessioniManagementUtils.loadImportFolder(dir);
+        folder.sort();
+
+        int clusterSize = 50;
+
+        int clusterNum=1;
+        File destDir =null;
+
+        for (Map.Entry<String, ArrayList<Concessione>> entry : folder.getRelationshipMap().entrySet()) {
+            String s = entry.getKey();
+            ArrayList<Concessione> l = entry.getValue();
+            if(destDir==null||destDir.list().length+l.size()>=clusterSize){
+                destDir = new File(dir, "cluster_" + clusterNum);
+                destDir.mkdirs();
+                clusterNum++;
+            }
+            File finalDestDir = destDir;
+            l.forEach(c -> {
+                try {
+                    Files.saveString(Serialization.write(c),new File(finalDestDir,c.getMongo_id()+".json").toPath());
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        }
+
+        System.out.println("Created "+clusterNum--+" sub folders");
+
+    }
+
+}

use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ConcessioniManagementUtils.java

@@ -0,0 +1,85 @@
+package org.gcube.application.cms.usecases.legacyConcessioni;
+
+import lombok.Data;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.gcube.application.geoportal.client.utils.Serialization;
+import org.gcube.application.geoportal.common.model.legacy.Concessione;
+import org.gcube.application.geoportal.common.utils.Files;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class ConcessioniManagementUtils {
+
+    @Data
+    public static class ImportFolder {
+        private Map<String, ArrayList<Concessione>> RelationshipMap=new HashMap<>();
+        private AtomicLong loadedCount=new AtomicLong(0l);
+
+        Comparator<Concessione> comparator = new Comparator<Concessione>() {
+            @Override
+            public int compare(Concessione o1, Concessione o2) {
+                return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
+            }
+        };
+
+        public String sort(){
+            StringBuilder reportBuilder=new StringBuilder();
+            // order lists
+            getRelationshipMap().forEach((s,l) ->{
+                log.info("Sorting {} ({} elements)",s,l.size());
+                Collections.sort(l,comparator);
+                l.forEach(concessione -> reportBuilder.append(concessione.getMongo_id()+ ",\"" +concessione.getNome()+"\","+ concessione.getDataInizioProgetto()+"\n"));
+            });
+            return reportBuilder.toString();
+        }
+    }
+
+
+
+    public static final ImportFolder loadImportFolder(File sourceDir){
+        ImportFolder toReturn = new ImportFolder();
+
+        for(File elementFolder:sourceDir.listFiles()){
+            if(elementFolder!=null&&!elementFolder.getName().equals("relationships.csv")) {
+
+                if (elementFolder.isFile() && elementFolder.getName().endsWith(".json")) {
+                    Concessione c = read(elementFolder);
+                    if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
+                        toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
+                    toReturn.getRelationshipMap().get(c.getNome()).add(c);
+                    toReturn.getLoadedCount().incrementAndGet();
+                } else
+                    try{
+
+                    for (File jsonFile : elementFolder.listFiles((dir1, name) -> {
+                        return name!=null&&name.endsWith(".json");
+                    })) {
+                        Concessione c = read(jsonFile);
+                        if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
+                            toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
+                        toReturn.getRelationshipMap().get(c.getNome()).add(c);
+                        toReturn.getLoadedCount().incrementAndGet();
+                    }
+                    }catch (Throwable t){
+                        log.warn("Error while using "+elementFolder.getAbsolutePath(),t);
+                    }
+            }
+        }
+
+        System.out.println("Loaded "+toReturn.getLoadedCount()+" elements from "+sourceDir.getAbsolutePath());
+        return toReturn;
+    }
+
+
+    @SneakyThrows
+    private static final Concessione read(File jsonFile) {
+        log.info("Reading "+jsonFile.getAbsolutePath());
+        String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
+        return Serialization.read(json,Concessione.class);
+    }
+}

use-cases/src/test/java/org/gcube/application/cms/usecases/ws/DescribeWSFolder.java

@@ -18,9 +18,10 @@ public class DescribeWSFolder {
     static StorageHubClient shc =null;
 
     public static void main(String[] args) throws StorageHubException {
-        String context="/gcube/devsec/devVRE";
+        // String context="/gcube/devsec/devVRE";
+        String context="/pred4s/preprod/preVRE";
 //        String folderID="0518e4ad-0ef9-4cba-8d4f-8e315146acdd";
-        String folderID="a0d5ef8d-8e6a-4dd7-8539-5a2293e852cd";
+        String folderID="ded73475-f9b1-46c8-94f8-5b35c60667ce";
         Boolean recursive = true;
 
         TokenSetter.set(context);