Utils
parent
235b47f595
commit
a7001b8100
@ -0,0 +1,49 @@
|
||||
package org.gcube.application.cms.usecases.legacyConcessioni;
|
||||
|
||||
import org.gcube.application.cms.serialization.Serialization;
|
||||
import org.gcube.application.geoportal.common.model.legacy.Concessione;
|
||||
import org.gcube.application.geoportal.common.utils.Files;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
|
||||
public class ClusterizeImportFolder {
|
||||
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
final File dir=new File("/Users/fabioisti/git/gcube-cms-suite/import1668778302348");
|
||||
|
||||
|
||||
ConcessioniManagementUtils.ImportFolder folder=ConcessioniManagementUtils.loadImportFolder(dir);
|
||||
folder.sort();
|
||||
|
||||
int clusterSize = 50;
|
||||
|
||||
int clusterNum=1;
|
||||
File destDir =null;
|
||||
|
||||
for (Map.Entry<String, ArrayList<Concessione>> entry : folder.getRelationshipMap().entrySet()) {
|
||||
String s = entry.getKey();
|
||||
ArrayList<Concessione> l = entry.getValue();
|
||||
if(destDir==null||destDir.list().length+l.size()>=clusterSize){
|
||||
destDir = new File(dir, "cluster_" + clusterNum);
|
||||
destDir.mkdirs();
|
||||
clusterNum++;
|
||||
}
|
||||
File finalDestDir = destDir;
|
||||
l.forEach(c -> {
|
||||
try {
|
||||
Files.saveString(Serialization.write(c),new File(finalDestDir,c.getMongo_id()+".json").toPath());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
System.out.println("Created "+clusterNum--+" sub folders");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
package org.gcube.application.cms.usecases.legacyConcessioni;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.gcube.application.geoportal.client.utils.Serialization;
|
||||
import org.gcube.application.geoportal.common.model.legacy.Concessione;
|
||||
import org.gcube.application.geoportal.common.utils.Files;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Slf4j
|
||||
public class ConcessioniManagementUtils {
|
||||
|
||||
@Data
|
||||
public static class ImportFolder {
|
||||
private Map<String, ArrayList<Concessione>> RelationshipMap=new HashMap<>();
|
||||
private AtomicLong loadedCount=new AtomicLong(0l);
|
||||
|
||||
Comparator<Concessione> comparator = new Comparator<Concessione>() {
|
||||
@Override
|
||||
public int compare(Concessione o1, Concessione o2) {
|
||||
return o1.getDataInizioProgetto().compareTo(o2.getDataInizioProgetto());
|
||||
}
|
||||
};
|
||||
|
||||
public String sort(){
|
||||
StringBuilder reportBuilder=new StringBuilder();
|
||||
// order lists
|
||||
getRelationshipMap().forEach((s,l) ->{
|
||||
log.info("Sorting {} ({} elements)",s,l.size());
|
||||
Collections.sort(l,comparator);
|
||||
l.forEach(concessione -> reportBuilder.append(concessione.getMongo_id()+ ",\"" +concessione.getNome()+"\","+ concessione.getDataInizioProgetto()+"\n"));
|
||||
});
|
||||
return reportBuilder.toString();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static final ImportFolder loadImportFolder(File sourceDir){
|
||||
ImportFolder toReturn = new ImportFolder();
|
||||
|
||||
for(File elementFolder:sourceDir.listFiles()){
|
||||
if(elementFolder!=null&&!elementFolder.getName().equals("relationships.csv")) {
|
||||
|
||||
if (elementFolder.isFile() && elementFolder.getName().endsWith(".json")) {
|
||||
Concessione c = read(elementFolder);
|
||||
if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
|
||||
toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
|
||||
toReturn.getRelationshipMap().get(c.getNome()).add(c);
|
||||
toReturn.getLoadedCount().incrementAndGet();
|
||||
} else
|
||||
try{
|
||||
|
||||
for (File jsonFile : elementFolder.listFiles((dir1, name) -> {
|
||||
return name!=null&&name.endsWith(".json");
|
||||
})) {
|
||||
Concessione c = read(jsonFile);
|
||||
if (!toReturn.getRelationshipMap().containsKey(c.getNome()))
|
||||
toReturn.getRelationshipMap().put(c.getNome(), new ArrayList<>());
|
||||
toReturn.getRelationshipMap().get(c.getNome()).add(c);
|
||||
toReturn.getLoadedCount().incrementAndGet();
|
||||
}
|
||||
}catch (Throwable t){
|
||||
log.warn("Error while using "+elementFolder.getAbsolutePath(),t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println("Loaded "+toReturn.getLoadedCount()+" elements from "+sourceDir.getAbsolutePath());
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private static final Concessione read(File jsonFile) {
|
||||
log.info("Reading "+jsonFile.getAbsolutePath());
|
||||
String json= Files.readFileAsString(jsonFile.getAbsolutePath(), Charset.defaultCharset());
|
||||
return Serialization.read(json,Concessione.class);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue