lucio.lelii
/
storagehub-scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added scritp for Mongo to min io porting

This commit is contained in:
Lucio Lelii 2022-11-21 16:15:01 +01:00
parent b27a314f9b
commit 78fb3d68f9
4 changed files with 206 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/CleanTrash.java
View File

@ -5,6 +5,7 @@ import java.util.List;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.PathNotFoundException;
import org.apache.jackrabbit.api.JackrabbitSession;
import org.gcube.common.storagehub.model.Path;
@ -20,11 +21,13 @@ import org.slf4j.LoggerFactory;
public class CleanTrash implements AbstractScript{
private static Logger log = LoggerFactory.getLogger(CleanTrash.class);
@Override
public String run(JackrabbitSession session, ScriptParameter prameters, ScriptUtil scriptUtil) {
StringBuilder stringBuilder = new StringBuilder();
try {
log.info("script started");
Node node = session.getNode("/Home");
NodeIterator it = node.getNodes();
@ -34,22 +37,35 @@ public class CleanTrash implements AbstractScript{
long start = System.currentTimeMillis();
Path workspacePath = Paths.append(Paths.getPath(home.getPath()),"Workspace");
if (!session.nodeExists(workspacePath.toPath())) continue;
Node trashNode = session.getNode(Paths.append(workspacePath, "Trash").toPath());
stringBuilder.append("\n").append("cleaning trash of ").append(home.getName());
Node trashNode = null;
try {
if (home.getProperty("hl:version").getLong()>0)
trashNode = session.getNode(Paths.append(Paths.getPath(home.getPath()), "Trash").toPath());
else
trashNode = session.getNode(Paths.append(workspacePath, "Trash").toPath());
}catch (PathNotFoundException e) {
stringBuilder.append("\n").append("path not found for ").append(home.getName());
log.warn("path not found for {} ",home.getName());
continue;
}
List<Item> items = scriptUtil.getChildren(null, trashNode, null, true, TrashItem.class);
for (Item item: items) {
TrashItem trashItem = (TrashItem) item;
stringBuilder.append("\n").append("removed node ").append(trashItem.getTitle());
if (System.currentTimeMillis() - trashItem.getDeletedTime().getTimeInMillis() > 2629800000l ) {
scriptUtil.removeNodes(session, Collections.singletonList(trashItem));
}
}
stringBuilder.append("\n").append("cleaned trash of ").append(home.getName()).append(" in ").append(System.currentTimeMillis()-start).append(" millis");
}
}catch (Exception e) {
StringWriter writer = new StringWriter();

52
src/main/java/GeneralScript.java
View File

@ -2,13 +2,20 @@ import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Collections;
import java.util.List;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.PathNotFoundException;
import javax.jcr.Property;
import javax.jcr.PropertyIterator;
import org.apache.jackrabbit.api.JackrabbitSession;
import org.gcube.common.storagehub.model.Path;
import org.gcube.common.storagehub.model.Paths;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.TrashItem;
import org.gcube.data.access.storagehub.scripting.AbstractScript;
import org.gcube.data.access.storagehub.scripting.ScriptParameter;
import org.gcube.data.access.storagehub.scripting.ScriptUtil;
@ -18,10 +25,45 @@ import org.slf4j.LoggerFactory;
public class GeneralScript implements AbstractScript{
private static Logger log = LoggerFactory.getLogger(GeneralScript.class);
@Override
public String run(JackrabbitSession session, ScriptParameter prameters, ScriptUtil scriptUtil) {
StringBuilder stringBuilder = new StringBuilder();
try {
log.info("script started");
Node home = session.getNode("/Home/emmanuel.blondel");
long start = System.currentTimeMillis();
stringBuilder.append("\n").append("cleaning trash of ").append(home.getName());
Node trashNode = null;
stringBuilder.append("\nversion of the trash is " ).append(home.getProperty("hl:version").getLong());
NodeIterator it = home.getNodes();
while (it.hasNext()) {
Node node = it.nextNode();
stringBuilder.append("\n").append(node.getPath());
}
}catch (Exception e) {
StringWriter writer = new StringWriter();
PrintWriter sw = new PrintWriter(writer, true);
e.printStackTrace(sw);
stringBuilder.append("\n").append(writer.toString());
}
return stringBuilder.toString();
/*StringBuilder stringBuilder = new StringBuilder();
try {
try(OutputStream os = new ByteArrayOutputStream()){
session.exportDocumentView("/", os, true, false);
@ -34,10 +76,10 @@ public class GeneralScript implements AbstractScript{
e.printStackTrace(sw);
return writer.toString();
}
return stringBuilder.toString();
return stringBuilder.toString();*/
}
private void visitChildren(StringBuilder stringBuilder, Node node, int level) throws Exception{
stringBuilder.append("-").append(node.getPath()).append("\n");
PropertyIterator props = node.getProperties();
@ -59,5 +101,5 @@ public class GeneralScript implements AbstractScript{
visitChildren(stringBuilder, child, level+1);
}
}
}

108
src/main/java/MongoToMinioPorting.java
View File

@ -1,4 +1,110 @@
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
public class MongoToMinioPorting {
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.PathNotFoundException;
import org.apache.jackrabbit.api.JackrabbitSession;
import org.gcube.common.storagehub.model.Constants;
import org.gcube.common.storagehub.model.Path;
import org.gcube.common.storagehub.model.Paths;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.TrashItem;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.data.access.storagehub.scripting.AbstractScript;
import org.gcube.data.access.storagehub.scripting.ScriptParameter;
import org.gcube.data.access.storagehub.scripting.ScriptUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MongoToMinioPorting implements AbstractScript{
private static Logger log = LoggerFactory.getLogger(MongoToMinioPorting.class);
@Override
public String run(JackrabbitSession session, ScriptParameter prameters, ScriptUtil scriptUtil) {
StringBuilder stringBuilder = new StringBuilder();
try {
Node node = session.getNode("/Home");
NodeIterator it = node.getNodes();
long startall = System.currentTimeMillis();
while (it.hasNext()) {
Node home = it.nextNode();
long start = System.currentTimeMillis();
Path workspacePath = Paths.append(Paths.getPath(home.getPath()),"Workspace");
if (!session.nodeExists(workspacePath.toPath())) continue;
List<Item> items = scriptUtil.getChildren(null, session.getNode(workspacePath.toPath()), null, true,null);
items.parallelStream().forEach(i -> stringBuilder.append(elaborateItem(i, session, scriptUtil)));
stringBuilder.append("\n").append("elaborated user home of ").append(home.getName()).append(" in ").append(System.currentTimeMillis()-start).append(" millis");
}
stringBuilder.append("\n").append("elaborated").append(" in ").append(System.currentTimeMillis()-startall).append(" millis");
}catch (Exception e) {
StringWriter writer = new StringWriter();
PrintWriter sw = new PrintWriter(writer, true);
e.printStackTrace(sw);
return writer.toString();
}
return stringBuilder.toString();
}
private String elaborateItem(Item item, JackrabbitSession session, ScriptUtil scriptUtil ) {
StringBuilder stringBuilder = new StringBuilder();
if (item instanceof FolderItem) {
FolderItem folder = (FolderItem) item;
try {
List<Item> items = scriptUtil.getChildren(null, (Node) folder.getRelatedNode(), null, true, null);
stringBuilder.append("\n").append("elaborating folder ").append(folder.getId()).append(" - ").append(folder.getTitle());
items.stream().forEach(i -> stringBuilder.append(elaborateItem(i, session, scriptUtil)));
}catch (Exception e) {
stringBuilder.append("\n").append("error on folder ").append(folder.getId()).append(" - ").append(folder.getTitle()).append(e.getMessage());
}
} else if (item instanceof AbstractFileItem) {
AbstractFileItem file = (AbstractFileItem) item;
if (file.getContent().getPayloadBackend().getStorageName()==Constants.MONGO_STORAGE) {
stringBuilder.append("\n").append("elaborating file ").append(file.getId()).append(" - ").append(file.getTitle());
try {
Collection<StorageBackendFactory> factories = scriptUtil.getStorageBackendHandler();
StorageBackend mongo;
StorageBackend minio;
for (StorageBackendFactory factory : factories) {
if (factory.getName() == Constants.MONGO_STORAGE)
mongo = factory.create(file.getContent().getPayloadBackend());
if (factory.getName() == Constants.DEFAULT_MINIO_STORAGE) {
minio = factory.create(new PayloadBackend(Constants.DEFAULT_MINIO_STORAGE, null));
}
}
//TODO: make real copy
//TODO: change content
stringBuilder.append("\n").append("file copied ").append(file.getId()).append(" - ").append(file.getTitle());
}catch (Exception e) {
stringBuilder.append("\n").append("error on file ").append(file.getId()).append(" - ").append(file.getTitle()).append(e.getMessage());
}
}
}
return stringBuilder.toString();
}
}

59
src/main/java/MoveVREsAndTrash.java
View File

@ -32,38 +32,38 @@ public class MoveVREsAndTrash implements AbstractScript{
Path workspacePath = Paths.append(Paths.getPath(home.getPath()),"Workspace");
Path homePath = Paths.getPath(home.getPath());
if (!session.nodeExists(workspacePath.toPath())) continue;
Boolean oldVresExists = session.nodeExists(Paths.append(workspacePath, "MySpecialFolders").toPath());
Boolean oldTrashExists = session.nodeExists(Paths.append(workspacePath, "Trash").toPath());
if (!(oldVresExists && oldTrashExists)) {
stringBuilder.append(home.getPath()).append(" not contains in Workspace folder: ");
if (!oldTrashExists) stringBuilder.append(" Trash ");
if (!oldVresExists) stringBuilder.append(" MySpecialFolders ");
stringBuilder.append("\n");
if (home.getProperty("hl:version").getLong()>0) {
stringBuilder.append(home.getPath()).append(" already updated\n");
continue;
}
session.getWorkspace().getLockManager().lock(workspacePath.toPath(), true, true, -1, "");
try {
if (home.getProperty("hl:version").getLong()>0) {
stringBuilder.append(home.getPath()).append(" already updated\n");
continue;
}
home.setProperty("hl:version", 1l);
session.move(Paths.append(workspacePath, "MySpecialFolders").toPath(), Paths.append(homePath, "VREs").toPath());
session.move(Paths.append(workspacePath, "Trash").toPath(), Paths.append(homePath, "Trash").toPath());
Boolean vresExist = session.nodeExists(Paths.append(homePath, "VREs").toPath());
Boolean trashExist = session.nodeExists(Paths.append(homePath, "Trash").toPath());
home.setProperty("hl:version", 1l);
if (oldVresExists)
session.move(Paths.append(workspacePath, "MySpecialFolders").toPath(), Paths.append(homePath, "VREs").toPath());
stringBuilder.append(workspacePath.toPath()).append(",").append(vresExist).append(",")
.append(trashExist).append(",").append(System.currentTimeMillis()-start).append("\n");
session.save();
/*
if (oldTrashExists)
session.move(Paths.append(workspacePath, "Trash").toPath(), Paths.append(homePath, "Trash").toPath());
else
home.addNode("Trash");
Boolean vresExist = session.nodeExists(Paths.append(homePath, "VREs").toPath());
Boolean trashExist = session.nodeExists(Paths.append(homePath, "Trash").toPath());
stringBuilder.append(workspacePath.toPath()).append(",").append(vresExist).append(",")
.append(trashExist).append(",").append(System.currentTimeMillis()-start).append("\n");
session.save();
/*
session.move("/Home/lucio.lelii/Workspace/MySpecialFolders", "/Home/lucio.lelii/VREs");
Node vres = session.getNode("/Home/lucio.lelii/VREs");
@ -73,15 +73,10 @@ public class MoveVREsAndTrash implements AbstractScript{
Node trash = session.getNode("/Home/lucio.lelii/Trash");
session.save();*/
} finally {
session.getWorkspace().getLockManager().unlock(workspacePath.toPath());
}
}
}catch (Exception e) {
}catch (Throwable e) {
StringWriter writer = new StringWriter();
PrintWriter sw = new PrintWriter(writer, true);
e.printStackTrace(sw);