lucio.lelii
/
storagehub-scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

script for minIO porting

This commit is contained in:
Lucio Lelii 2022-12-05 15:45:02 +01:00
parent 5a4aa31298
commit c774e1e849
3 changed files with 458 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
src/main/java/MongoToMinIOPortingSharedFolder.java Normal file
View File

@ -0,0 +1,225 @@
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.Session;
import javax.jcr.lock.LockManager;
import javax.jcr.version.Version;
import javax.jcr.version.VersionHistory;
import javax.jcr.version.VersionIterator;
import javax.jcr.version.VersionManager;
import org.apache.jackrabbit.api.JackrabbitSession;
import org.gcube.common.storagehub.model.Constants;
import org.gcube.common.storagehub.model.NodeConstants;
import org.gcube.common.storagehub.model.exceptions.StorageIdNotFoundException;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.SharedFolder;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.data.access.storagehub.scripting.AbstractScript;
import org.gcube.data.access.storagehub.scripting.ScriptParameter;
import org.gcube.data.access.storagehub.scripting.ScriptUtil;
public class MongoToMinIOPortingSharedFolder implements AbstractScript{
//private static Logger log = LoggerFactory.getLogger(MongoToMinioPortingNotShared.class);
@Override
public String run(JackrabbitSession session, ScriptParameter prameters, ScriptUtil scriptUtil) {
StringBuilder stringBuilder = new StringBuilder();
try {
long startall = System.currentTimeMillis();
for (Node shared: sharedFolders(session)) {
stringBuilder.append("\n").append("elaborating sharedFolder ").append(shared.getPath());
long start = System.currentTimeMillis();
Item item = scriptUtil.getItem(shared, null);
elaborateItem(item, session, scriptUtil, stringBuilder);
stringBuilder.append("\n").append("elaborated sharedFolder ").append(shared.getPath()).append(" in ").append(System.currentTimeMillis()-start).append(" millis");
}
stringBuilder.append("\n").append(" TOTAL elaborated").append(" in ").append(System.currentTimeMillis()-startall).append(" millis");
}catch (Exception e) {
StringWriter writer = new StringWriter();
PrintWriter sw = new PrintWriter(writer, true);
e.printStackTrace(sw);
return writer.toString();
}
return stringBuilder.toString();
}
private void elaborateItem(Item item, JackrabbitSession session, ScriptUtil scriptUtil, StringBuilder stringBuilder ) {
if (item instanceof FolderItem) {
FolderItem folder = (FolderItem) item;
if (folder.isShared() && ((SharedFolder)folder).isVreFolder()) {
stringBuilder.append("\n").append("skipped VREFolder ").append(folder.getTitle());
return;
}
try {
List<Item> items = scriptUtil.getChildren(null, (Node) folder.getRelatedNode(), null, true, null);
stringBuilder.append("\n").append("elaborating folder ").append(folder.getId()).append(" - ").append(folder.getTitle());
items.stream().forEach(i -> elaborateItem(i, session, scriptUtil, stringBuilder));
}catch (Exception e) {
stringBuilder.append("\n").append("error on folder ").append(folder.getId()).append(" - ").append(folder.getTitle()).append(e.getMessage());
}
} else
if (item instanceof AbstractFileItem) {
try {
AbstractFileItem file = (AbstractFileItem) item;
Node contentNode = ((Node) item.getRelatedNode()).getNode(NodeConstants.CONTENT_NAME);
Collection<StorageBackendFactory> factories = scriptUtil.getStorageBackendHandler();
StorageBackend mongo = null;
StorageBackend minio= null;
for (StorageBackendFactory factory : factories) {
if (factory.getName() == Constants.MONGO_STORAGE)
mongo = factory.create(file.getContent().getPayloadBackend());
if (factory.getName() == Constants.DEFAULT_MINIO_STORAGE) {
minio = factory.create(new PayloadBackend(Constants.DEFAULT_MINIO_STORAGE, null));
}
}
if (file.getContent().getPayloadBackend().getStorageName()==Constants.MONGO_STORAGE) {
stringBuilder.append("\n").append("elaborating file ").append(file.getId()).append(" - ").append(file.getTitle());
try {
String mongoId = file.getContent().getStorageId();
stringBuilder.append("\n").append("searching for content id ").append(mongoId);
MetaInfo info;
try (InputStream is = mongo.download(mongoId);){
info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize(), file.getOwner() );
}catch (StorageIdNotFoundException e) {
stringBuilder.append("\n").append("not found content id ").append(mongoId);
throw e;
}
Content content = ((AbstractFileItem) item).getContent();
content.setPayloadBackend(info.getPayloadBackend());
content.setStorageId(info.getStorageId());
boolean canBeDeleted = true;
LockManager lockMan = session.getWorkspace().getLockManager();
lockMan.lock(((Node) item.getRelatedNode()).getPath(), true, true, -1, "");
try {
checkoutContentNode(contentNode);
scriptUtil.updateContentNode(content, contentNode );
stringBuilder.append("\n").append("file copied ").append(file.getId()).append(" - ").append(file.getPath());
session.save();
} catch (Throwable e) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
stringBuilder.append("\n").append("ERROR updating content ").append(file.getId()).append(" - ").append(file.getPath()).append(" || ").append(sw.toString());
canBeDeleted = false;
} finally {
if (lockMan.isLocked(((Node) item.getRelatedNode()).getPath()))
lockMan.unlock(((Node) item.getRelatedNode()).getPath());
}
if (canBeDeleted)
mongo.delete(mongoId);
else minio.delete(info.getStorageId());
}catch (Throwable e) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
stringBuilder.append("\n").append("error on file ").append(file.getId()).append(" - ").append(file.getTitle()).append(" || ").append(sw.toString());;
}
}
try {
//replace versions
for (Version version : getVersions(contentNode)) {
try {
stringBuilder.append("\n").append("checking version ").append(version.getName()).append(" - ").append(file.getPath());
String name = version.getName();
Node related = version.getFrozenNode();
if (related.hasProperty("hl:storageId") && !related.hasProperty("hl:payloadBackend")) {
String versionMongoId = related.getProperty("hl:storageId").getString();
long size = related.getProperty("hl:size").getLong();
stringBuilder.append("\n").append("searching on version for content id ").append(versionMongoId);
try (InputStream stream = mongo.download(versionMongoId)){
minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size, file.getOwner());
}
stringBuilder.append("\n").append("version copied ").append(name).append(" - ").append(file.getPath());
}
}catch (Exception e) {
stringBuilder.append("\n").append("ERROR copying version ").append(version.getName()).append(" - ").append(file.getPath());
}
}
}catch (Throwable e) {
stringBuilder.append("\n").append("ERROR getting versions for ").append(file.getPath()).append(" - ").append(file.getId());
}
}catch (Throwable e) {
stringBuilder.append("\n").append("error TOTAL on file ").append(item.getTitle()).append(e.getMessage());
}
}
}
private List<Node> sharedFolders(JackrabbitSession session) throws Exception{
Node shares = session.getNode("/Share");
NodeIterator it = shares.getNodes();
List<Node> sharesNodes = new ArrayList<>();
while (it.hasNext()) {
Node shared = it.nextNode();
sharesNodes.add(shared);
}
return sharesNodes;
}
/* private void checkinContentNode(Node node) throws Exception{
Session session = node.getSession();
VersionManager versionManager = session.getWorkspace().getVersionManager();
versionManager.checkin(node.getPath());
}*/
private void checkoutContentNode(Node node) throws Exception{
Session session = node.getSession();
VersionManager versionManager = session.getWorkspace().getVersionManager();
versionManager.checkout(node.getPath());
}
private List<Version> getVersions(Node node) throws Exception{
Session session = node.getSession();
VersionManager versionManager = session.getWorkspace().getVersionManager();
VersionHistory history = versionManager.getVersionHistory(node.getPath());
VersionIterator nodeIt = history.getAllLinearVersions();
List<Version> versionedNodes = new ArrayList<Version>();
while (nodeIt.hasNext()) {
Version version = nodeIt.nextVersion();
if (!versionManager.getBaseVersion(node.getPath()).isSame(version))
versionedNodes.add(version);
}
return versionedNodes;
}
}

231
src/main/java/MongoToMinIOPortingVREFolders.java Normal file
View File

@ -0,0 +1,231 @@
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.Session;
import javax.jcr.lock.LockManager;
import javax.jcr.version.Version;
import javax.jcr.version.VersionHistory;
import javax.jcr.version.VersionIterator;
import javax.jcr.version.VersionManager;
import org.apache.jackrabbit.api.JackrabbitSession;
import org.gcube.common.storagehub.model.Constants;
import org.gcube.common.storagehub.model.NodeConstants;
import org.gcube.common.storagehub.model.exceptions.StorageIdNotFoundException;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.SharedFolder;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.items.nodes.PayloadBackend;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.data.access.storagehub.scripting.AbstractScript;
import org.gcube.data.access.storagehub.scripting.ScriptParameter;
import org.gcube.data.access.storagehub.scripting.ScriptUtil;
public class MongoToMinIOPortingVREFolders implements AbstractScript{
//private static Logger log = LoggerFactory.getLogger(MongoToMinioPortingNotShared.class);
@Override
public String run(JackrabbitSession session, ScriptParameter prameters, ScriptUtil scriptUtil) {
StringBuilder stringBuilder = new StringBuilder();
try {
long startall = System.currentTimeMillis();
for (Node shared: sharedFolders(session)) {
long start = System.currentTimeMillis();
Item folder = scriptUtil.getItem(shared, null);
if (! (folder instanceof SharedFolder))
continue;
if (!((SharedFolder)folder).isVreFolder()) {
continue;
}
stringBuilder.append("\n").append("elaborating VREFolder ").append(shared.getPath());
Item item = scriptUtil.getItem(shared, null);
elaborateItem(item, session, scriptUtil, stringBuilder);
stringBuilder.append("\n").append("elaborated VREFolder ").append(shared.getPath()).append(" in ").append(System.currentTimeMillis()-start).append(" millis");
}
stringBuilder.append("\n").append(" TOTAL elaborated").append(" in ").append(System.currentTimeMillis()-startall).append(" millis");
}catch (Exception e) {
StringWriter writer = new StringWriter();
PrintWriter sw = new PrintWriter(writer, true);
e.printStackTrace(sw);
return writer.toString();
}
return stringBuilder.toString();
}
private void elaborateItem(Item item, JackrabbitSession session, ScriptUtil scriptUtil, StringBuilder stringBuilder ) {
if (item instanceof FolderItem) {
FolderItem folder = (FolderItem) item;
try {
List<Item> items = scriptUtil.getChildren(null, (Node) folder.getRelatedNode(), null, true, null);
stringBuilder.append("\n").append("elaborating folder ").append(folder.getId()).append(" - ").append(folder.getTitle());
items.stream().forEach(i -> elaborateItem(i, session, scriptUtil, stringBuilder));
}catch (Exception e) {
stringBuilder.append("\n").append("error on folder ").append(folder.getId()).append(" - ").append(folder.getTitle()).append(e.getMessage());
}
} else
if (item instanceof AbstractFileItem) {
try {
AbstractFileItem file = (AbstractFileItem) item;
Node contentNode = ((Node) item.getRelatedNode()).getNode(NodeConstants.CONTENT_NAME);
Collection<StorageBackendFactory> factories = scriptUtil.getStorageBackendHandler();
StorageBackend mongo = null;
StorageBackend minio= null;
for (StorageBackendFactory factory : factories) {
if (factory.getName() == Constants.MONGO_STORAGE)
mongo = factory.create(file.getContent().getPayloadBackend());
if (factory.getName() == Constants.DEFAULT_MINIO_STORAGE) {
minio = factory.create(new PayloadBackend(Constants.DEFAULT_MINIO_STORAGE, null));
}
}
if (file.getContent().getPayloadBackend().getStorageName()==Constants.MONGO_STORAGE) {
stringBuilder.append("\n").append("elaborating file ").append(file.getId()).append(" - ").append(file.getTitle());
try {
String mongoId = file.getContent().getStorageId();
stringBuilder.append("\n").append("searching for content id ").append(mongoId);
MetaInfo info;
try (InputStream is = mongo.download(mongoId);){
info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize() , file.getOwner() );
}catch (StorageIdNotFoundException e) {
stringBuilder.append("\n").append("not found content id ").append(mongoId);
throw e;
}
Content content = ((AbstractFileItem) item).getContent();
content.setPayloadBackend(info.getPayloadBackend());
content.setStorageId(info.getStorageId());
boolean canBeDeleted = true;
LockManager lockMan = session.getWorkspace().getLockManager();
lockMan.lock(((Node) item.getRelatedNode()).getPath(), true, true, -1, "");
try {
checkoutContentNode(contentNode);
scriptUtil.updateContentNode(content, contentNode );
stringBuilder.append("\n").append("file copied ").append(file.getId()).append(" - ").append(file.getPath());
session.save();
} catch (Throwable e) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
stringBuilder.append("\n").append("ERROR updating content ").append(file.getId()).append(" - ").append(file.getPath()).append(" || ").append(sw.toString());
canBeDeleted = false;
} finally {
if (lockMan.isLocked(((Node) item.getRelatedNode()).getPath()))
lockMan.unlock(((Node) item.getRelatedNode()).getPath());
}
if (canBeDeleted)
mongo.delete(mongoId);
else minio.delete(info.getStorageId());
}catch (Throwable e) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
stringBuilder.append("\n").append("error on file ").append(file.getId()).append(" - ").append(file.getTitle()).append(" || ").append(sw.toString());;
}
}
try {
//replace versions
for (Version version : getVersions(contentNode)) {
try {
stringBuilder.append("\n").append("checking version ").append(version.getName()).append(" - ").append(file.getPath());
String name = version.getName();
Node related = version.getFrozenNode();
if (related.hasProperty("hl:storageId") && !related.hasProperty("hl:payloadBackend")) {
String versionMongoId = related.getProperty("hl:storageId").getString();
long size = related.getProperty("hl:size").getLong();
stringBuilder.append("\n").append("searching on version for content id ").append(versionMongoId);
try (InputStream stream = mongo.download(versionMongoId)){
minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size , file.getOwner());
}
stringBuilder.append("\n").append("version copied ").append(name).append(" - ").append(file.getPath());
}
}catch (Exception e) {
stringBuilder.append("\n").append("ERROR copying version ").append(version.getName()).append(" - ").append(file.getPath());
}
}
}catch (Throwable e) {
stringBuilder.append("\n").append("ERROR getting versions for ").append(file.getPath()).append(" - ").append(file.getId());
}
}catch (Throwable e) {
stringBuilder.append("\n").append("error TOTAL on file ").append(item.getTitle()).append(e.getMessage());
}
}
}
private List<Node> sharedFolders(JackrabbitSession session) throws Exception{
Node shares = session.getNode("/Share");
NodeIterator it = shares.getNodes();
List<Node> sharesNodes = new ArrayList<>();
while (it.hasNext()) {
Node shared = it.nextNode();
sharesNodes.add(shared);
}
return sharesNodes;
}
/* private void checkinContentNode(Node node) throws Exception{
Session session = node.getSession();
VersionManager versionManager = session.getWorkspace().getVersionManager();
versionManager.checkin(node.getPath());
}*/
private void checkoutContentNode(Node node) throws Exception{
Session session = node.getSession();
VersionManager versionManager = session.getWorkspace().getVersionManager();
versionManager.checkout(node.getPath());
}
private List<Version> getVersions(Node node) throws Exception{
Session session = node.getSession();
VersionManager versionManager = session.getWorkspace().getVersionManager();
VersionHistory history = versionManager.getVersionHistory(node.getPath());
VersionIterator nodeIt = history.getAllLinearVersions();
List<Version> versionedNodes = new ArrayList<Version>();
while (nodeIt.hasNext()) {
Version version = nodeIt.nextVersion();
if (!versionManager.getBaseVersion(node.getPath()).isSame(version))
versionedNodes.add(version);
}
return versionedNodes;
}
}

4
src/main/java/MongoToMinioPortingNotShared.java
View File

@ -106,7 +106,7 @@ public class MongoToMinioPortingNotShared implements AbstractScript{
MetaInfo info;
try (InputStream is = mongo.download(mongoId);){
info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize() );
info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize(), file.getOwner() );
}catch (StorageIdNotFoundException e) {
stringBuilder.append("\n").append("not found content id ").append(mongoId);
throw e;
@ -161,7 +161,7 @@ public class MongoToMinioPortingNotShared implements AbstractScript{
stringBuilder.append("\n").append("searching on version for content id ").append(versionMongoId);
try (InputStream stream = mongo.download(versionMongoId)){
minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size);
minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size , file.getOwner());
}
stringBuilder.append("\n").append("version copied ").append(name).append(" - ").append(file.getPath());
}