diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java index 98e4fb9..06f22a0 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java @@ -2,14 +2,15 @@ package org.gcube.application.geoportal.common.utils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; +import org.xml.sax.InputSource; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.net.URL; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.*; @Slf4j @@ -112,4 +113,9 @@ public class Files { if(is!=null) IOUtils.closeQuietly(is); } } + + + public static final void saveString(String content, Path dest) throws IOException { + java.nio.file.Files.copy(new ByteArrayInputStream(content.getBytes()),dest, StandardCopyOption.REPLACE_EXISTING); + } } diff --git a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/tests/GCubeTest.java b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/tests/GCubeTest.java index 2b54862..bcf8d7e 100644 --- a/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/tests/GCubeTest.java +++ b/geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/tests/GCubeTest.java @@ -11,12 +11,11 @@ public class GCubeTest { } -// testContext = "/pred4s/preprod/preVRE"; // InterfaceConstants.SERVICE_CLASS="Application"; // InterfaceConstants.SERVICE_NAME="GeoPortal"; - testContext = "/gcube/devsec/devVRE"; - + // testContext = "/pred4s/preprod/preVRE"; + testContext= "/gcube/devsec/devVRE"; System.out.println("TEST CONTEXT = "+testContext); return testContext; diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/ClearProjects.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/ClearProjects.java index 8a3321c..6619389 100644 --- a/use-cases/src/test/java/org/gcube/application/cms/usecases/ClearProjects.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/ClearProjects.java @@ -1,10 +1,12 @@ package org.gcube.application.cms.usecases; import lombok.extern.slf4j.Slf4j; +import org.bson.Document; import org.gcube.application.cms.tests.TokenSetter; import org.gcube.application.geoportal.common.model.document.Project; import org.gcube.application.geoportal.common.model.rest.QueryRequest; import org.gcube.application.geoportal.common.rest.Projects; +import org.gcube.application.geoportal.common.utils.tests.GCubeTest; import java.rmi.RemoteException; import java.util.ArrayList; @@ -21,11 +23,11 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl public class ClearProjects { public static void main(String[] args) throws RemoteException, InterruptedException { - String context="/gcube/devsec/devVRE"; + // String context="/gcube/devsec/devVRE"; - //String context="/pred4s/preprod/preVRE"; + // String context="/pred4s/preprod/preVRE"; - TokenSetter.set(context); + TokenSetter.set(GCubeTest.getContext()); Projects client=projects("profiledConcessioni").build();; @@ -42,8 +44,13 @@ public class ClearProjects { AtomicLong found=new AtomicLong(0); Iterator it=null; - it=client.query(new QueryRequest()); -// it=manager.search("{\"nome\" : {$regex : \"Mock .*\"}, \"creationTime\" :{$gt : \"2021-10-18T13:58:53.326\"}}"); + + String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"Landro (Comune di Tambre, BL) \"},\"_theDocument.dataInizioProgetto\":{\"$eq\":\"2021-05-29T00:00:00\"}}\n" + + "}"); + QueryRequest q = new QueryRequest(); + q.setFilter(Document.parse(queryString)); + it=client.query(q); + ExecutorService service = Executors.newFixedThreadPool(1); @@ -63,7 +70,7 @@ public class ClearProjects { @Override public void run() { try{ - TokenSetter.set(context); + TokenSetter.set(GCubeTest.getContext()); String currentId=c.getId(); if(currentId==null) { System.out.println("ID IS NULL " + c); diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/ExportConcessioniAsProjects.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/ExportConcessioniAsProjects.java index 38b6775..3917758 100644 --- a/use-cases/src/test/java/org/gcube/application/cms/usecases/ExportConcessioniAsProjects.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/ExportConcessioniAsProjects.java @@ -1,30 +1,37 @@ package org.gcube.application.cms.usecases; -import lombok.SneakyThrows; +import com.fasterxml.jackson.core.JsonProcessingException; import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione; +import org.gcube.application.cms.serialization.Serialization; import org.gcube.application.cms.tests.TokenSetter; -import org.gcube.application.geoportal.client.utils.Serialization; +import org.gcube.application.cms.usecases.legacyConcessioni.ConcessioniManagementUtils; import org.gcube.application.geoportal.common.model.configuration.Configuration; import org.gcube.application.geoportal.common.model.document.Project; import org.gcube.application.geoportal.common.model.document.access.Access; import org.gcube.application.geoportal.common.model.document.access.AccessPolicy; +import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation; import org.gcube.application.geoportal.common.model.legacy.*; import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest; +import org.gcube.application.geoportal.common.model.rest.QueryRequest; import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest; import org.gcube.application.geoportal.common.model.rest.TempFile; import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field; import org.gcube.application.geoportal.common.rest.Projects; import org.gcube.application.geoportal.common.utils.FileSets; import org.gcube.application.geoportal.common.utils.Files; +import org.gcube.application.geoportal.common.utils.tests.GCubeTest; import java.io.File; -import java.nio.charset.Charset; +import java.io.IOException; import java.rmi.RemoteException; +import java.time.Instant; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -35,23 +42,27 @@ public class ExportConcessioniAsProjects { - @SneakyThrows - private static final Concessione read(File jsonFile) { - log.info("Reading "+jsonFile.getAbsolutePath()); - String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset()); - return Serialization.read(json,Concessione.class); - } - public static void main(String[] args) throws InterruptedException, RemoteException { + + public static void main(String[] args) throws InterruptedException, IOException { // read from imported folder - File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1666713419018"); + File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348"); + + // select only cluster +// dir = new File(dir,"cluster_1"); + + + + File errorDir=new File(dir,"errors_"+ Instant.now().toString()); + + Boolean pushProjects=true; // File dir= new File("test-data/test-packages/related_concessioni"); // File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378"); - String targetContext="/gcube/devsec/devVRE"; - + //String targetContext="/gcube/devsec/devVRE"; + // String targetContext="/pred4s/preprod/preVRE"; Projects client=projects("profiledConcessioni").build();; @@ -59,47 +70,19 @@ public class ExportConcessioniAsProjects { Integer totalCount = 0; - Comparator comparator = new Comparator() { - @Override - public int compare(Concessione o1, Concessione o2) { - return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto()); - } - }; + // Title - > Time ordered list - AtomicLong loadedCount = new AtomicLong(0); - Map> relationshipMap = new HashMap<>(); - for(File elementFolder:dir.listFiles()){ - if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) { - Concessione c =read(elementFolder); - if(!relationshipMap.containsKey(c.getNome())) - relationshipMap.put(c.getNome(),new ArrayList<>()); - relationshipMap.get(c.getNome()).add(c); - loadedCount.incrementAndGet(); - } - else - for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})){ - Concessione c =read(jsonFile); - if(!relationshipMap.containsKey(c.getNome())) - relationshipMap.put(c.getNome(),new ArrayList<>()); - relationshipMap.get(c.getNome()).add(c); - loadedCount.incrementAndGet(); - } + ConcessioniManagementUtils.ImportFolder importFolder = ConcessioniManagementUtils.loadImportFolder(dir); + + Files.saveString(importFolder.sort(),new File(dir,"relationships.csv").toPath()); + + if(!pushProjects){ + System.out.println("PUSH PROJECTS FLAG IS FALSE.. CLOSING."); + System.exit(0); } - System.out.println("Loaded "+loadedCount+" elements from "+dir.getAbsolutePath()); - - // order lists - relationshipMap.forEach((s,l) ->{ - log.info("Sorting {} ({} elements)",s,l.size()); - Collections.sort(l,comparator); - if(l.size()>1) { - l.forEach(concessione -> log.info("{} : {} ",concessione.getMongo_id(),concessione.getDataInizioProgetto())); - } - }); - - - TokenSetter.set(targetContext); + TokenSetter.set(GCubeTest.getContext()); // StorageUtils storage = new StorageUtils(); @@ -107,38 +90,63 @@ public class ExportConcessioniAsProjects { AtomicLong warnCount = new AtomicLong(0); AtomicLong errCount = new AtomicLong(0); - ExecutorService service = Executors.newFixedThreadPool(1); + ExecutorService service = Executors.newFixedThreadPool(3); long startProcess = System.currentTimeMillis(); + importFolder.getRelationshipMap().forEach((s,l)->{ - relationshipMap.forEach((s,l)->{ - String relationshipTarget=null; - for (Concessione c : l) { - - String finalRelationshipTarget = relationshipTarget; - CompletableFuture lastPublished = CompletableFuture.supplyAsync(new Supplier() { - @Override - public String get() { - return publish(c, client, errCount, count, finalRelationshipTarget); + AtomicReference isSetOk = new AtomicReference<>(true); + // Check if exists + Concessione c = l.get(0); + try { + List existing= getExisting(c,client); + if(existing.isEmpty()) { + log.info("Not Found"); + }else { + existing.forEach(project -> { + if (!project.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK)) + isSetOk.set(false); + }); + if(!isSetOk.get()){ + log.debug("Deleting error set for {}",c.getNome()); + for (Project project : existing) { + client.deleteById(project.getId()); + } } - },service); - try { - relationshipTarget = lastPublished.get(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (ExecutionException e) { - throw new RuntimeException(e); } + }catch(NullPointerException e){} + catch (Throwable t){throw new RuntimeException("Unexpected Exception while checking for "+c.getNome());} + + if(!isSetOk.get()) { + String relationshipTarget = null; + for (Concessione concessione : l) { + + String finalRelationshipTarget = relationshipTarget; + CompletableFuture lastPublished = CompletableFuture.supplyAsync(new Supplier() { + @Override + public String get() { + return publish(concessione, client, errCount, count, finalRelationshipTarget, errorDir); + } + }, service); + try { + relationshipTarget = lastPublished.get(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + + } } }); while (!service.awaitTermination(1, TimeUnit.MINUTES)) { - log.info("Waiting .. completed {}, out of {} ",count.get(),loadedCount); - if(loadedCount.get()==count.get()) service.shutdown(); + log.info("Waiting .. completed {}, out of {} ",count.get(),importFolder.getLoadedCount()); + if(importFolder.getLoadedCount().get()==count.get()) service.shutdown(); } System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]"); @@ -160,7 +168,7 @@ public class ExportConcessioniAsProjects { - private static String publish(Concessione c, Projects client, AtomicLong errCount, AtomicLong count, String relationshipTarget){ + private static String publish(Concessione c, Projects client, AtomicLong errCount, AtomicLong count, String relationshipTarget, File errorDir){ Project project = null; try { long startTime = System.currentTimeMillis(); @@ -168,6 +176,8 @@ public class ExportConcessioniAsProjects { log.info("Using {} {}",c.getNome(),c.getMongo_id()); + + //Copy core fields Document doc=new Document(); @@ -374,6 +384,17 @@ public class ExportConcessioniAsProjects { } catch (Throwable throwable) { System.err.println(throwable); errCount.incrementAndGet(); + String ser = null; + File destFile=null; + //if(relationshipTarget!=null) Files.saveString(); + if(project!=null) { + ser = org.gcube.application.cms.serialization.Serialization.write(project); + destFile = new File(errorDir,"project_"+ project.getId()); + }else{ + ser = org.gcube.application.cms.serialization.Serialization.write(c); + destFile = new File(errorDir,"conc_"+ c.getMongo_id()); + } + Files.saveString(ser, destFile.toPath()); } finally { count.incrementAndGet(); return project!=null? project.getId() : null; @@ -401,4 +422,28 @@ public class ExportConcessioniAsProjects { return toReturn; } + + private static List getExisting(Concessione c,Projects client) throws RemoteException, JsonProcessingException, NullPointerException { + try { + QueryRequest req = new QueryRequest(); + String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"%1$s\"}}", c.getNome()); + log.debug("Query String is {}", queryString); + req.setFilter(Document.parse(queryString)); + AtomicInteger count = new AtomicInteger(0); + StringBuilder msg = new StringBuilder(); + ArrayList toReturn = new ArrayList<>(); + + client.query(req).forEachRemaining(p -> { + Project proj = (Project) p; + count.incrementAndGet(); + toReturn.add(proj); + msg.append(proj.getId() + ","); + }); + log.debug("Found {} for {}", count.get(), c.getNome()); + return toReturn; + }catch(Throwable t){ + log.warn("ERROR while querying ",t); + throw t; + } + } } diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClearConcessioni.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClearConcessioni.java index aff09f3..e708c12 100644 --- a/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClearConcessioni.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClearConcessioni.java @@ -18,9 +18,9 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl public class ClearConcessioni { public static void main(String[] args) throws Exception { - String context="/gcube/devsec/devVRE"; + // String context="/gcube/devsec/devVRE"; - //String context="/pred4s/preprod/preVRE"; + String context="/pred4s/preprod/preVRE"; TokenSetter.set(context); diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClusterizeImportFolder.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClusterizeImportFolder.java new file mode 100644 index 0000000..e2e69ec --- /dev/null +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClusterizeImportFolder.java @@ -0,0 +1,49 @@ +package org.gcube.application.cms.usecases.legacyConcessioni; + +import org.gcube.application.cms.serialization.Serialization; +import org.gcube.application.geoportal.common.model.legacy.Concessione; +import org.gcube.application.geoportal.common.utils.Files; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +public class ClusterizeImportFolder { + + + public static void main(String[] args) throws IOException { + final File dir=new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348"); + + + ConcessioniManagementUtils.ImportFolder folder=ConcessioniManagementUtils.loadImportFolder(dir); + folder.sort(); + + int clusterSize = 50; + + int clusterNum=1; + File destDir =null; + + for (Map.Entry> entry : folder.getRelationshipMap().entrySet()) { + String s = entry.getKey(); + ArrayList l = entry.getValue(); + if(destDir==null||destDir.list().length+l.size()>=clusterSize){ + destDir = new File(dir, "cluster_" + clusterNum); + destDir.mkdirs(); + clusterNum++; + } + File finalDestDir = destDir; + l.forEach(c -> { + try { + Files.saveString(Serialization.write(c),new File(finalDestDir,c.getMongo_id()+".json").toPath()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + System.out.println("Created "+clusterNum--+" sub folders"); + + } + +} diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ConcessioniManagementUtils.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ConcessioniManagementUtils.java new file mode 100644 index 0000000..9b6b369 --- /dev/null +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ConcessioniManagementUtils.java @@ -0,0 +1,85 @@ +package org.gcube.application.cms.usecases.legacyConcessioni; + +import lombok.Data; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.gcube.application.geoportal.client.utils.Serialization; +import org.gcube.application.geoportal.common.model.legacy.Concessione; +import org.gcube.application.geoportal.common.utils.Files; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +public class ConcessioniManagementUtils { + + @Data + public static class ImportFolder { + private Map> RelationshipMap=new HashMap<>(); + private AtomicLong loadedCount=new AtomicLong(0l); + + Comparator comparator = new Comparator() { + @Override + public int compare(Concessione o1, Concessione o2) { + return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto()); + } + }; + + public String sort(){ + StringBuilder reportBuilder=new StringBuilder(); + // order lists + getRelationshipMap().forEach((s,l) ->{ + log.info("Sorting {} ({} elements)",s,l.size()); + Collections.sort(l,comparator); + l.forEach(concessione -> reportBuilder.append(concessione.getMongo_id()+ ",\"" +concessione.getNome()+"\","+ concessione.getDataInizioProgetto()+"\n")); + }); + return reportBuilder.toString(); + } + } + + + + public static final ImportFolder loadImportFolder(File sourceDir){ + ImportFolder toReturn = new ImportFolder(); + + for(File elementFolder:sourceDir.listFiles()){ + if(elementFolder!=null&&!elementFolder.getName().equals("relationships.csv")) { + + if (elementFolder.isFile() && elementFolder.getName().endsWith(".json")) { + Concessione c = read(elementFolder); + if (!toReturn.getRelationshipMap().containsKey(c.getNome())) + toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>()); + toReturn.getRelationshipMap().get(c.getNome()).add(c); + toReturn.getLoadedCount().incrementAndGet(); + } else + try{ + + for (File jsonFile : elementFolder.listFiles((dir1, name) -> { + return name!=null&&name.endsWith(".json"); + })) { + Concessione c = read(jsonFile); + if (!toReturn.getRelationshipMap().containsKey(c.getNome())) + toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>()); + toReturn.getRelationshipMap().get(c.getNome()).add(c); + toReturn.getLoadedCount().incrementAndGet(); + } + }catch (Throwable t){ + log.warn("Error while using "+elementFolder.getAbsolutePath(),t); + } + } + } + + System.out.println("Loaded "+toReturn.getLoadedCount()+" elements from "+sourceDir.getAbsolutePath()); + return toReturn; + } + + + @SneakyThrows + private static final Concessione read(File jsonFile) { + log.info("Reading "+jsonFile.getAbsolutePath()); + String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset()); + return Serialization.read(json,Concessione.class); + } +} diff --git a/use-cases/src/test/java/org/gcube/application/cms/usecases/ws/DescribeWSFolder.java b/use-cases/src/test/java/org/gcube/application/cms/usecases/ws/DescribeWSFolder.java index a434c22..3247a3d 100644 --- a/use-cases/src/test/java/org/gcube/application/cms/usecases/ws/DescribeWSFolder.java +++ b/use-cases/src/test/java/org/gcube/application/cms/usecases/ws/DescribeWSFolder.java @@ -18,9 +18,10 @@ public class DescribeWSFolder { static StorageHubClient shc =null; public static void main(String[] args) throws StorageHubException { - String context="/gcube/devsec/devVRE"; + // String context="/gcube/devsec/devVRE"; + String context="/pred4s/preprod/preVRE"; // String folderID="0518e4ad-0ef9-4cba-8d4f-8e315146acdd"; - String folderID="a0d5ef8d-8e6a-4dd7-8539-5a2293e852cd"; + String folderID="ded73475-f9b1-46c8-94f8-5b35c60667ce"; Boolean recursive = true; TokenSetter.set(context);