gCubeSystem
/
gcube-cms-suite
14
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Utils

This commit is contained in:
Fabio Sinibaldi 2022-11-21 20:51:15 +01:00
parent 235b47f595
commit a7001b8100
8 changed files with 278 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/Files.java
View File

@ -2,14 +2,15 @@ package org.gcube.application.geoportal.common.utils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.xml.sax.InputSource;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.*;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.CopyOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.*;
@Slf4j
@ -112,4 +113,9 @@ public class Files {
if(is!=null) IOUtils.closeQuietly(is);
}
}
public static final void saveString(String content, Path dest) throws IOException {
java.nio.file.Files.copy(new ByteArrayInputStream(content.getBytes()),dest, StandardCopyOption.REPLACE_EXISTING);
}
}

3
geoportal-common/src/main/java/org/gcube/application/geoportal/common/utils/tests/GCubeTest.java
View File

@ -11,13 +11,12 @@ public class GCubeTest {
}
// testContext = "/pred4s/preprod/preVRE";
// InterfaceConstants.SERVICE_CLASS="Application";
// InterfaceConstants.SERVICE_NAME="GeoPortal";
// testContext = "/pred4s/preprod/preVRE";
testContext= "/gcube/devsec/devVRE";
System.out.println("TEST CONTEXT = "+testContext);
return testContext;
}

17
use-cases/src/test/java/org/gcube/application/cms/usecases/ClearProjects.java
View File

@ -1,10 +1,12 @@
package org.gcube.application.cms.usecases;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.tests.TokenSetter;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.rest.Projects;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import java.rmi.RemoteException;
import java.util.ArrayList;
@ -21,11 +23,11 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl
public class ClearProjects {
public static void main(String[] args) throws RemoteException, InterruptedException {
String context="/gcube/devsec/devVRE";
// String context="/gcube/devsec/devVRE";
// String context="/pred4s/preprod/preVRE";
TokenSetter.set(context);
TokenSetter.set(GCubeTest.getContext());
Projects<Project> client=projects("profiledConcessioni").build();;
@ -42,8 +44,13 @@ public class ClearProjects {
AtomicLong found=new AtomicLong(0);
Iterator<Project> it=null;
it=client.query(new QueryRequest());
// it=manager.search("{\"nome\" : {$regex : \"Mock .*\"}, \"creationTime\" :{$gt : \"2021-10-18T13:58:53.326\"}}");
String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"Landro (Comune di Tambre, BL) \"},\"_theDocument.dataInizioProgetto\":{\"$eq\":\"2021-05-29T00:00:00\"}}\n" +
"}");
QueryRequest q = new QueryRequest();
q.setFilter(Document.parse(queryString));
it=client.query(q);
ExecutorService service = Executors.newFixedThreadPool(1);
@ -63,7 +70,7 @@ public class ClearProjects {
@Override
public void run() {
try{
TokenSetter.set(context);
TokenSetter.set(GCubeTest.getContext());
String currentId=c.getId();
if(currentId==null) {
System.out.println("ID IS NULL " + c);

159
use-cases/src/test/java/org/gcube/application/cms/usecases/ExportConcessioniAsProjects.java
View File

@ -1,30 +1,37 @@
package org.gcube.application.cms.usecases;
import lombok.SneakyThrows;
import com.fasterxml.jackson.core.JsonProcessingException;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.gcube.application.cms.custom.gna.concessioni.model.ProfiledConcessione;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.cms.tests.TokenSetter;
import org.gcube.application.geoportal.client.utils.Serialization;
import org.gcube.application.cms.usecases.legacyConcessioni.ConcessioniManagementUtils;
import org.gcube.application.geoportal.common.model.configuration.Configuration;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.access.Access;
import org.gcube.application.geoportal.common.model.document.access.AccessPolicy;
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
import org.gcube.application.geoportal.common.model.legacy.*;
import org.gcube.application.geoportal.common.model.rest.CreateRelationshipRequest;
import org.gcube.application.geoportal.common.model.rest.QueryRequest;
import org.gcube.application.geoportal.common.model.rest.StepExecutionRequest;
import org.gcube.application.geoportal.common.model.rest.TempFile;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.Field;
import org.gcube.application.geoportal.common.rest.Projects;
import org.gcube.application.geoportal.common.utils.FileSets;
import org.gcube.application.geoportal.common.utils.Files;
import org.gcube.application.geoportal.common.utils.tests.GCubeTest;
import java.io.File;
import java.nio.charset.Charset;
import java.io.IOException;
import java.rmi.RemoteException;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -35,23 +42,27 @@ public class ExportConcessioniAsProjects {
@SneakyThrows
private static final Concessione read(File jsonFile) {
log.info("Reading "+jsonFile.getAbsolutePath());
String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
return Serialization.read(json,Concessione.class);
}
public static void main(String[] args) throws InterruptedException, RemoteException {
public static void main(String[] args) throws InterruptedException, IOException {
// read from imported folder
File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1666713419018");
File dir= new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348");
// select only cluster
// dir = new File(dir,"cluster_1");
File errorDir=new File(dir,"errors_"+ Instant.now().toString());
Boolean pushProjects=true;
// File dir= new File("test-data/test-packages/related_concessioni");
// File dir = new File("/Users/fabioisti/Documents/Work/GNA DATA/Bug_23378");
String targetContext="/gcube/devsec/devVRE";
//String targetContext="/gcube/devsec/devVRE";
// String targetContext="/pred4s/preprod/preVRE";
Projects<Project> client=projects("profiledConcessioni").build();;
@ -59,47 +70,19 @@ public class ExportConcessioniAsProjects {
Integer totalCount = 0;
Comparator<Concessione> comparator = new Comparator<Concessione>() {
@Override
public int compare(Concessione o1, Concessione o2) {
return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
}
};
// Title - > Time ordered list
AtomicLong loadedCount = new AtomicLong(0);
Map<String, ArrayList<Concessione>> relationshipMap = new HashMap<>();
for(File elementFolder:dir.listFiles()){
if(elementFolder.isFile()&&elementFolder.getName().endsWith(".json")) {
Concessione c =read(elementFolder);
if(!relationshipMap.containsKey(c.getNome()))
relationshipMap.put(c.getNome(),new ArrayList<>());
relationshipMap.get(c.getNome()).add(c);
loadedCount.incrementAndGet();
}
else
for(File jsonFile:elementFolder.listFiles((dir1, name) -> {return name.endsWith(".json");})){
Concessione c =read(jsonFile);
if(!relationshipMap.containsKey(c.getNome()))
relationshipMap.put(c.getNome(),new ArrayList<>());
relationshipMap.get(c.getNome()).add(c);
loadedCount.incrementAndGet();
}
ConcessioniManagementUtils.ImportFolder importFolder = ConcessioniManagementUtils.loadImportFolder(dir);
Files.saveString(importFolder.sort(),new File(dir,"relationships.csv").toPath());
if(!pushProjects){
System.out.println("PUSH PROJECTS FLAG IS FALSE.. CLOSING.");
System.exit(0);
}
System.out.println("Loaded "+loadedCount+" elements from "+dir.getAbsolutePath());
// order lists
relationshipMap.forEach((s,l) ->{
log.info("Sorting {} ({} elements)",s,l.size());
Collections.sort(l,comparator);
if(l.size()>1) {
l.forEach(concessione -> log.info("{} : {} ",concessione.getMongo_id(),concessione.getDataInizioProgetto()));
}
});
TokenSetter.set(targetContext);
TokenSetter.set(GCubeTest.getContext());
// StorageUtils storage = new StorageUtils();
@ -107,20 +90,44 @@ public class ExportConcessioniAsProjects {
AtomicLong warnCount = new AtomicLong(0);
AtomicLong errCount = new AtomicLong(0);
ExecutorService service = Executors.newFixedThreadPool(1);
ExecutorService service = Executors.newFixedThreadPool(3);
long startProcess = System.currentTimeMillis();
importFolder.getRelationshipMap().forEach((s,l)->{
relationshipMap.forEach((s,l)->{
AtomicReference<Boolean> isSetOk = new AtomicReference<>(true);
// Check if exists
Concessione c = l.get(0);
try {
List<Project> existing= getExisting(c,client);
if(existing.isEmpty()) {
log.info("Not Found");
}else {
existing.forEach(project -> {
if (!project.getLifecycleInformation().getLastOperationStatus().equals(LifecycleInformation.Status.OK))
isSetOk.set(false);
});
if(!isSetOk.get()){
log.debug("Deleting error set for {}",c.getNome());
for (Project project : existing) {
client.deleteById(project.getId());
}
}
}
}catch(NullPointerException e){}
catch (Throwable t){throw new RuntimeException("Unexpected Exception while checking for "+c.getNome());}
if(!isSetOk.get()) {
String relationshipTarget = null;
for (Concessione c : l) {
for (Concessione concessione : l) {
String finalRelationshipTarget = relationshipTarget;
CompletableFuture<String> lastPublished = CompletableFuture.supplyAsync(new Supplier<String>() {
@Override
public String get() {
return publish(c, client, errCount, count, finalRelationshipTarget);
return publish(concessione, client, errCount, count, finalRelationshipTarget, errorDir);
}
}, service);
try {
@ -132,13 +139,14 @@ public class ExportConcessioniAsProjects {
}
}
}
});
while (!service.awaitTermination(1, TimeUnit.MINUTES)) {
log.info("Waiting .. completed {}, out of {} ",count.get(),loadedCount);
if(loadedCount.get()==count.get()) service.shutdown();
log.info("Waiting .. completed {}, out of {} ",count.get(),importFolder.getLoadedCount());
if(importFolder.getLoadedCount().get()==count.get()) service.shutdown();
}
System.out.println("Completed "+count.get()+" [elapsedTime = "+(System.currentTimeMillis()-startProcess)/1000+" sec] [warn : "+warnCount.get()+", err : "+errCount.get()+"]");
@ -160,7 +168,7 @@ public class ExportConcessioniAsProjects {
private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget){
private static String publish(Concessione c, Projects<Project> client, AtomicLong errCount, AtomicLong count, String relationshipTarget, File errorDir){
Project project = null;
try {
long startTime = System.currentTimeMillis();
@ -168,6 +176,8 @@ public class ExportConcessioniAsProjects {
log.info("Using {} {}",c.getNome(),c.getMongo_id());
//Copy core fields
Document doc=new Document();
@ -374,6 +384,17 @@ public class ExportConcessioniAsProjects {
} catch (Throwable throwable) {
System.err.println(throwable);
errCount.incrementAndGet();
String ser = null;
File destFile=null;
//if(relationshipTarget!=null) Files.saveString();
if(project!=null) {
ser = org.gcube.application.cms.serialization.Serialization.write(project);
destFile = new File(errorDir,"project_"+ project.getId());
}else{
ser = org.gcube.application.cms.serialization.Serialization.write(c);
destFile = new File(errorDir,"conc_"+ c.getMongo_id());
}
Files.saveString(ser, destFile.toPath());
} finally {
count.incrementAndGet();
return project!=null? project.getId() : null;
@ -401,4 +422,28 @@ public class ExportConcessioniAsProjects {
return toReturn;
}
private static List<Project> getExisting(Concessione c,Projects client) throws RemoteException, JsonProcessingException, NullPointerException {
try {
QueryRequest req = new QueryRequest();
String queryString = String.format("{\"_theDocument.nome\" :{\"$eq\" : \"%1$s\"}}", c.getNome());
log.debug("Query String is {}", queryString);
req.setFilter(Document.parse(queryString));
AtomicInteger count = new AtomicInteger(0);
StringBuilder msg = new StringBuilder();
ArrayList<Project> toReturn = new ArrayList<>();
client.query(req).forEachRemaining(p -> {
Project proj = (Project) p;
count.incrementAndGet();
toReturn.add(proj);
msg.append(proj.getId() + ",");
});
log.debug("Found {} for {}", count.get(), c.getNome());
return toReturn;
}catch(Throwable t){
log.warn("ERROR while querying ",t);
throw t;
}
}
}

4
use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClearConcessioni.java
View File

@ -18,9 +18,9 @@ import static org.gcube.application.geoportal.client.plugins.GeoportalAbstractPl
public class ClearConcessioni {
public static void main(String[] args) throws Exception {
String context="/gcube/devsec/devVRE";
// String context="/gcube/devsec/devVRE";
//String context="/pred4s/preprod/preVRE";
String context="/pred4s/preprod/preVRE";
TokenSetter.set(context);

49
use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ClusterizeImportFolder.java Normal file
View File

@ -0,0 +1,49 @@
package org.gcube.application.cms.usecases.legacyConcessioni;
import org.gcube.application.cms.serialization.Serialization;
import org.gcube.application.geoportal.common.model.legacy.Concessione;
import org.gcube.application.geoportal.common.utils.Files;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
public class ClusterizeImportFolder {
public static void main(String[] args) throws IOException {
final File dir=new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348");
ConcessioniManagementUtils.ImportFolder folder=ConcessioniManagementUtils.loadImportFolder(dir);
folder.sort();
int clusterSize = 50;
int clusterNum=1;
File destDir =null;
for (Map.Entry<String, ArrayList<Concessione>> entry : folder.getRelationshipMap().entrySet()) {
String s = entry.getKey();
ArrayList<Concessione> l = entry.getValue();
if(destDir==null||destDir.list().length+l.size()>=clusterSize){
destDir = new File(dir, "cluster_" + clusterNum);
destDir.mkdirs();
clusterNum++;
}
File finalDestDir = destDir;
l.forEach(c -> {
try {
Files.saveString(Serialization.write(c),new File(finalDestDir,c.getMongo_id()+".json").toPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
System.out.println("Created "+clusterNum--+" sub folders");
}
}

85
use-cases/src/test/java/org/gcube/application/cms/usecases/legacyConcessioni/ConcessioniManagementUtils.java Normal file
View File

@ -0,0 +1,85 @@
package org.gcube.application.cms.usecases.legacyConcessioni;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.gcube.application.geoportal.client.utils.Serialization;
import org.gcube.application.geoportal.common.model.legacy.Concessione;
import org.gcube.application.geoportal.common.utils.Files;
import java.io.File;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class ConcessioniManagementUtils {
@Data
public static class ImportFolder {
private Map<String, ArrayList<Concessione>> RelationshipMap=new HashMap<>();
private AtomicLong loadedCount=new AtomicLong(0l);
Comparator<Concessione> comparator = new Comparator<Concessione>() {
@Override
public int compare(Concessione o1, Concessione o2) {
return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
}
};
public String sort(){
StringBuilder reportBuilder=new StringBuilder();
// order lists
getRelationshipMap().forEach((s,l) ->{
log.info("Sorting {} ({} elements)",s,l.size());
Collections.sort(l,comparator);
l.forEach(concessione -> reportBuilder.append(concessione.getMongo_id()+ ",\"" +concessione.getNome()+"\","+ concessione.getDataInizioProgetto()+"\n"));
});
return reportBuilder.toString();
}
}
public static final ImportFolder loadImportFolder(File sourceDir){
ImportFolder toReturn = new ImportFolder();
for(File elementFolder:sourceDir.listFiles()){
if(elementFolder!=null&&!elementFolder.getName().equals("relationships.csv")) {
if (elementFolder.isFile() && elementFolder.getName().endsWith(".json")) {
Concessione c = read(elementFolder);
if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
toReturn.getRelationshipMap().get(c.getNome()).add(c);
toReturn.getLoadedCount().incrementAndGet();
} else
try{
for (File jsonFile : elementFolder.listFiles((dir1, name) -> {
return name!=null&&name.endsWith(".json");
})) {
Concessione c = read(jsonFile);
if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
toReturn.getRelationshipMap().get(c.getNome()).add(c);
toReturn.getLoadedCount().incrementAndGet();
}
}catch (Throwable t){
log.warn("Error while using "+elementFolder.getAbsolutePath(),t);
}
}
}
System.out.println("Loaded "+toReturn.getLoadedCount()+" elements from "+sourceDir.getAbsolutePath());
return toReturn;
}
@SneakyThrows
private static final Concessione read(File jsonFile) {
log.info("Reading "+jsonFile.getAbsolutePath());
String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
return Serialization.read(json,Concessione.class);
}
}

5
use-cases/src/test/java/org/gcube/application/cms/usecases/ws/DescribeWSFolder.java
View File

@ -18,9 +18,10 @@ public class DescribeWSFolder {
static StorageHubClient shc =null;
public static void main(String[] args) throws StorageHubException {
String context="/gcube/devsec/devVRE";
// String context="/gcube/devsec/devVRE";
String context="/pred4s/preprod/preVRE";
// String folderID="0518e4ad-0ef9-4cba-8d4f-8e315146acdd";
String folderID="a0d5ef8d-8e6a-4dd7-8539-5a2293e852cd";
String folderID="ded73475-f9b1-46c8-94f8-5b35c60667ce";
Boolean recursive = true;
TokenSetter.set(context);