diff --git a/src/main/java/MongoToMinIOPortingSharedFolder.java b/src/main/java/MongoToMinIOPortingSharedFolder.java new file mode 100644 index 0000000..3dad02a --- /dev/null +++ b/src/main/java/MongoToMinIOPortingSharedFolder.java @@ -0,0 +1,225 @@ +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.jcr.Node; +import javax.jcr.NodeIterator; +import javax.jcr.Session; +import javax.jcr.lock.LockManager; +import javax.jcr.version.Version; +import javax.jcr.version.VersionHistory; +import javax.jcr.version.VersionIterator; +import javax.jcr.version.VersionManager; + +import org.apache.jackrabbit.api.JackrabbitSession; +import org.gcube.common.storagehub.model.Constants; +import org.gcube.common.storagehub.model.NodeConstants; +import org.gcube.common.storagehub.model.exceptions.StorageIdNotFoundException; +import org.gcube.common.storagehub.model.items.AbstractFileItem; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; +import org.gcube.common.storagehub.model.items.SharedFolder; +import org.gcube.common.storagehub.model.items.nodes.Content; +import org.gcube.common.storagehub.model.items.nodes.PayloadBackend; +import org.gcube.common.storagehub.model.storages.MetaInfo; +import org.gcube.common.storagehub.model.storages.StorageBackend; +import org.gcube.common.storagehub.model.storages.StorageBackendFactory; +import org.gcube.data.access.storagehub.scripting.AbstractScript; +import org.gcube.data.access.storagehub.scripting.ScriptParameter; +import org.gcube.data.access.storagehub.scripting.ScriptUtil; + +public class MongoToMinIOPortingSharedFolder implements AbstractScript{ + + //private static Logger log = LoggerFactory.getLogger(MongoToMinioPortingNotShared.class); + + @Override + public String run(JackrabbitSession session, ScriptParameter prameters, ScriptUtil scriptUtil) { + StringBuilder stringBuilder = new StringBuilder(); + try { + long startall = System.currentTimeMillis(); + + for (Node shared: sharedFolders(session)) { + stringBuilder.append("\n").append("elaborating sharedFolder ").append(shared.getPath()); + long start = System.currentTimeMillis(); + + Item item = scriptUtil.getItem(shared, null); + elaborateItem(item, session, scriptUtil, stringBuilder); + + stringBuilder.append("\n").append("elaborated sharedFolder ").append(shared.getPath()).append(" in ").append(System.currentTimeMillis()-start).append(" millis"); + + } + + stringBuilder.append("\n").append(" TOTAL elaborated").append(" in ").append(System.currentTimeMillis()-startall).append(" millis"); + }catch (Exception e) { + StringWriter writer = new StringWriter(); + PrintWriter sw = new PrintWriter(writer, true); + e.printStackTrace(sw); + return writer.toString(); + } + return stringBuilder.toString(); + } + + + private void elaborateItem(Item item, JackrabbitSession session, ScriptUtil scriptUtil, StringBuilder stringBuilder ) { + if (item instanceof FolderItem) { + FolderItem folder = (FolderItem) item; + if (folder.isShared() && ((SharedFolder)folder).isVreFolder()) { + stringBuilder.append("\n").append("skipped VREFolder ").append(folder.getTitle()); + return; + } + try { + List items = scriptUtil.getChildren(null, (Node) folder.getRelatedNode(), null, true, null); + stringBuilder.append("\n").append("elaborating folder ").append(folder.getId()).append(" - ").append(folder.getTitle()); + items.stream().forEach(i -> elaborateItem(i, session, scriptUtil, stringBuilder)); + + }catch (Exception e) { + stringBuilder.append("\n").append("error on folder ").append(folder.getId()).append(" - ").append(folder.getTitle()).append(e.getMessage()); + } + + } else + if (item instanceof AbstractFileItem) { + try { + AbstractFileItem file = (AbstractFileItem) item; + Node contentNode = ((Node) item.getRelatedNode()).getNode(NodeConstants.CONTENT_NAME); + + + Collection factories = scriptUtil.getStorageBackendHandler(); + StorageBackend mongo = null; + StorageBackend minio= null; + for (StorageBackendFactory factory : factories) { + if (factory.getName() == Constants.MONGO_STORAGE) + mongo = factory.create(file.getContent().getPayloadBackend()); + if (factory.getName() == Constants.DEFAULT_MINIO_STORAGE) { + minio = factory.create(new PayloadBackend(Constants.DEFAULT_MINIO_STORAGE, null)); + } + } + + if (file.getContent().getPayloadBackend().getStorageName()==Constants.MONGO_STORAGE) { + stringBuilder.append("\n").append("elaborating file ").append(file.getId()).append(" - ").append(file.getTitle()); + try { + String mongoId = file.getContent().getStorageId(); + stringBuilder.append("\n").append("searching for content id ").append(mongoId); + MetaInfo info; + + try (InputStream is = mongo.download(mongoId);){ + info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize(), file.getOwner() ); + }catch (StorageIdNotFoundException e) { + stringBuilder.append("\n").append("not found content id ").append(mongoId); + throw e; + } + Content content = ((AbstractFileItem) item).getContent(); + + content.setPayloadBackend(info.getPayloadBackend()); + content.setStorageId(info.getStorageId()); + + + + boolean canBeDeleted = true; + + LockManager lockMan = session.getWorkspace().getLockManager(); + lockMan.lock(((Node) item.getRelatedNode()).getPath(), true, true, -1, ""); + try { + checkoutContentNode(contentNode); + scriptUtil.updateContentNode(content, contentNode ); + stringBuilder.append("\n").append("file copied ").append(file.getId()).append(" - ").append(file.getPath()); + session.save(); + } catch (Throwable e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + stringBuilder.append("\n").append("ERROR updating content ").append(file.getId()).append(" - ").append(file.getPath()).append(" || ").append(sw.toString()); + + canBeDeleted = false; + } finally { + if (lockMan.isLocked(((Node) item.getRelatedNode()).getPath())) + lockMan.unlock(((Node) item.getRelatedNode()).getPath()); + } + + if (canBeDeleted) + mongo.delete(mongoId); + else minio.delete(info.getStorageId()); + + }catch (Throwable e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + stringBuilder.append("\n").append("error on file ").append(file.getId()).append(" - ").append(file.getTitle()).append(" || ").append(sw.toString());; + } + } + try { + //replace versions + for (Version version : getVersions(contentNode)) { + try { + stringBuilder.append("\n").append("checking version ").append(version.getName()).append(" - ").append(file.getPath()); + String name = version.getName(); + Node related = version.getFrozenNode(); + if (related.hasProperty("hl:storageId") && !related.hasProperty("hl:payloadBackend")) { + String versionMongoId = related.getProperty("hl:storageId").getString(); + long size = related.getProperty("hl:size").getLong(); + stringBuilder.append("\n").append("searching on version for content id ").append(versionMongoId); + + try (InputStream stream = mongo.download(versionMongoId)){ + minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size, file.getOwner()); + } + stringBuilder.append("\n").append("version copied ").append(name).append(" - ").append(file.getPath()); + } + }catch (Exception e) { + stringBuilder.append("\n").append("ERROR copying version ").append(version.getName()).append(" - ").append(file.getPath()); + } + } + }catch (Throwable e) { + stringBuilder.append("\n").append("ERROR getting versions for ").append(file.getPath()).append(" - ").append(file.getId()); + } + }catch (Throwable e) { + stringBuilder.append("\n").append("error TOTAL on file ").append(item.getTitle()).append(e.getMessage()); + } + } + } + + + private List sharedFolders(JackrabbitSession session) throws Exception{ + Node shares = session.getNode("/Share"); + + NodeIterator it = shares.getNodes(); + + List sharesNodes = new ArrayList<>(); + while (it.hasNext()) { + + Node shared = it.nextNode(); + sharesNodes.add(shared); + } + + return sharesNodes; + } + +/* private void checkinContentNode(Node node) throws Exception{ + Session session = node.getSession(); + VersionManager versionManager = session.getWorkspace().getVersionManager(); + versionManager.checkin(node.getPath()); + }*/ + + private void checkoutContentNode(Node node) throws Exception{ + Session session = node.getSession(); + VersionManager versionManager = session.getWorkspace().getVersionManager(); + versionManager.checkout(node.getPath()); + } + + + private List getVersions(Node node) throws Exception{ + Session session = node.getSession(); + VersionManager versionManager = session.getWorkspace().getVersionManager(); + + VersionHistory history = versionManager.getVersionHistory(node.getPath()); + VersionIterator nodeIt = history.getAllLinearVersions(); + List versionedNodes = new ArrayList(); + while (nodeIt.hasNext()) { + Version version = nodeIt.nextVersion(); + if (!versionManager.getBaseVersion(node.getPath()).isSame(version)) + versionedNodes.add(version); + } + + return versionedNodes; + } + +} diff --git a/src/main/java/MongoToMinIOPortingVREFolders.java b/src/main/java/MongoToMinIOPortingVREFolders.java new file mode 100644 index 0000000..99a6265 --- /dev/null +++ b/src/main/java/MongoToMinIOPortingVREFolders.java @@ -0,0 +1,231 @@ +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.jcr.Node; +import javax.jcr.NodeIterator; +import javax.jcr.Session; +import javax.jcr.lock.LockManager; +import javax.jcr.version.Version; +import javax.jcr.version.VersionHistory; +import javax.jcr.version.VersionIterator; +import javax.jcr.version.VersionManager; + +import org.apache.jackrabbit.api.JackrabbitSession; +import org.gcube.common.storagehub.model.Constants; +import org.gcube.common.storagehub.model.NodeConstants; +import org.gcube.common.storagehub.model.exceptions.StorageIdNotFoundException; +import org.gcube.common.storagehub.model.items.AbstractFileItem; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; +import org.gcube.common.storagehub.model.items.SharedFolder; +import org.gcube.common.storagehub.model.items.nodes.Content; +import org.gcube.common.storagehub.model.items.nodes.PayloadBackend; +import org.gcube.common.storagehub.model.storages.MetaInfo; +import org.gcube.common.storagehub.model.storages.StorageBackend; +import org.gcube.common.storagehub.model.storages.StorageBackendFactory; +import org.gcube.data.access.storagehub.scripting.AbstractScript; +import org.gcube.data.access.storagehub.scripting.ScriptParameter; +import org.gcube.data.access.storagehub.scripting.ScriptUtil; + +public class MongoToMinIOPortingVREFolders implements AbstractScript{ + + //private static Logger log = LoggerFactory.getLogger(MongoToMinioPortingNotShared.class); + + @Override + public String run(JackrabbitSession session, ScriptParameter prameters, ScriptUtil scriptUtil) { + StringBuilder stringBuilder = new StringBuilder(); + try { + long startall = System.currentTimeMillis(); + + for (Node shared: sharedFolders(session)) { + + long start = System.currentTimeMillis(); + + Item folder = scriptUtil.getItem(shared, null); + if (! (folder instanceof SharedFolder)) + continue; + + if (!((SharedFolder)folder).isVreFolder()) { + continue; + } + + stringBuilder.append("\n").append("elaborating VREFolder ").append(shared.getPath()); + + Item item = scriptUtil.getItem(shared, null); + elaborateItem(item, session, scriptUtil, stringBuilder); + + stringBuilder.append("\n").append("elaborated VREFolder ").append(shared.getPath()).append(" in ").append(System.currentTimeMillis()-start).append(" millis"); + + } + + stringBuilder.append("\n").append(" TOTAL elaborated").append(" in ").append(System.currentTimeMillis()-startall).append(" millis"); + }catch (Exception e) { + StringWriter writer = new StringWriter(); + PrintWriter sw = new PrintWriter(writer, true); + e.printStackTrace(sw); + return writer.toString(); + } + return stringBuilder.toString(); + } + + + private void elaborateItem(Item item, JackrabbitSession session, ScriptUtil scriptUtil, StringBuilder stringBuilder ) { + if (item instanceof FolderItem) { + FolderItem folder = (FolderItem) item; + try { + List items = scriptUtil.getChildren(null, (Node) folder.getRelatedNode(), null, true, null); + stringBuilder.append("\n").append("elaborating folder ").append(folder.getId()).append(" - ").append(folder.getTitle()); + items.stream().forEach(i -> elaborateItem(i, session, scriptUtil, stringBuilder)); + + }catch (Exception e) { + stringBuilder.append("\n").append("error on folder ").append(folder.getId()).append(" - ").append(folder.getTitle()).append(e.getMessage()); + } + + } else + if (item instanceof AbstractFileItem) { + try { + AbstractFileItem file = (AbstractFileItem) item; + Node contentNode = ((Node) item.getRelatedNode()).getNode(NodeConstants.CONTENT_NAME); + + + Collection factories = scriptUtil.getStorageBackendHandler(); + StorageBackend mongo = null; + StorageBackend minio= null; + for (StorageBackendFactory factory : factories) { + if (factory.getName() == Constants.MONGO_STORAGE) + mongo = factory.create(file.getContent().getPayloadBackend()); + if (factory.getName() == Constants.DEFAULT_MINIO_STORAGE) { + minio = factory.create(new PayloadBackend(Constants.DEFAULT_MINIO_STORAGE, null)); + } + } + + if (file.getContent().getPayloadBackend().getStorageName()==Constants.MONGO_STORAGE) { + stringBuilder.append("\n").append("elaborating file ").append(file.getId()).append(" - ").append(file.getTitle()); + try { + String mongoId = file.getContent().getStorageId(); + stringBuilder.append("\n").append("searching for content id ").append(mongoId); + MetaInfo info; + + try (InputStream is = mongo.download(mongoId);){ + info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize() , file.getOwner() ); + }catch (StorageIdNotFoundException e) { + stringBuilder.append("\n").append("not found content id ").append(mongoId); + throw e; + } + Content content = ((AbstractFileItem) item).getContent(); + + content.setPayloadBackend(info.getPayloadBackend()); + content.setStorageId(info.getStorageId()); + + + + boolean canBeDeleted = true; + + LockManager lockMan = session.getWorkspace().getLockManager(); + lockMan.lock(((Node) item.getRelatedNode()).getPath(), true, true, -1, ""); + try { + checkoutContentNode(contentNode); + scriptUtil.updateContentNode(content, contentNode ); + stringBuilder.append("\n").append("file copied ").append(file.getId()).append(" - ").append(file.getPath()); + session.save(); + } catch (Throwable e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + stringBuilder.append("\n").append("ERROR updating content ").append(file.getId()).append(" - ").append(file.getPath()).append(" || ").append(sw.toString()); + + canBeDeleted = false; + } finally { + if (lockMan.isLocked(((Node) item.getRelatedNode()).getPath())) + lockMan.unlock(((Node) item.getRelatedNode()).getPath()); + } + + if (canBeDeleted) + mongo.delete(mongoId); + else minio.delete(info.getStorageId()); + + }catch (Throwable e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + stringBuilder.append("\n").append("error on file ").append(file.getId()).append(" - ").append(file.getTitle()).append(" || ").append(sw.toString());; + } + } + try { + //replace versions + for (Version version : getVersions(contentNode)) { + try { + stringBuilder.append("\n").append("checking version ").append(version.getName()).append(" - ").append(file.getPath()); + String name = version.getName(); + Node related = version.getFrozenNode(); + if (related.hasProperty("hl:storageId") && !related.hasProperty("hl:payloadBackend")) { + String versionMongoId = related.getProperty("hl:storageId").getString(); + long size = related.getProperty("hl:size").getLong(); + stringBuilder.append("\n").append("searching on version for content id ").append(versionMongoId); + + try (InputStream stream = mongo.download(versionMongoId)){ + minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size , file.getOwner()); + } + stringBuilder.append("\n").append("version copied ").append(name).append(" - ").append(file.getPath()); + } + }catch (Exception e) { + stringBuilder.append("\n").append("ERROR copying version ").append(version.getName()).append(" - ").append(file.getPath()); + } + } + }catch (Throwable e) { + stringBuilder.append("\n").append("ERROR getting versions for ").append(file.getPath()).append(" - ").append(file.getId()); + } + }catch (Throwable e) { + stringBuilder.append("\n").append("error TOTAL on file ").append(item.getTitle()).append(e.getMessage()); + } + } + } + + + private List sharedFolders(JackrabbitSession session) throws Exception{ + Node shares = session.getNode("/Share"); + + NodeIterator it = shares.getNodes(); + + List sharesNodes = new ArrayList<>(); + while (it.hasNext()) { + + Node shared = it.nextNode(); + sharesNodes.add(shared); + } + + return sharesNodes; + } + +/* private void checkinContentNode(Node node) throws Exception{ + Session session = node.getSession(); + VersionManager versionManager = session.getWorkspace().getVersionManager(); + versionManager.checkin(node.getPath()); + }*/ + + private void checkoutContentNode(Node node) throws Exception{ + Session session = node.getSession(); + VersionManager versionManager = session.getWorkspace().getVersionManager(); + versionManager.checkout(node.getPath()); + } + + + private List getVersions(Node node) throws Exception{ + Session session = node.getSession(); + VersionManager versionManager = session.getWorkspace().getVersionManager(); + + VersionHistory history = versionManager.getVersionHistory(node.getPath()); + VersionIterator nodeIt = history.getAllLinearVersions(); + List versionedNodes = new ArrayList(); + while (nodeIt.hasNext()) { + Version version = nodeIt.nextVersion(); + if (!versionManager.getBaseVersion(node.getPath()).isSame(version)) + versionedNodes.add(version); + } + + return versionedNodes; + } + +} \ No newline at end of file diff --git a/src/main/java/MongoToMinioPortingNotShared.java b/src/main/java/MongoToMinioPortingNotShared.java index 790ad84..950d46c 100644 --- a/src/main/java/MongoToMinioPortingNotShared.java +++ b/src/main/java/MongoToMinioPortingNotShared.java @@ -106,7 +106,7 @@ public class MongoToMinioPortingNotShared implements AbstractScript{ MetaInfo info; try (InputStream is = mongo.download(mongoId);){ - info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize() ); + info = minio.upload(is, file.getContent().getRemotePath() , item.getName(), mongoId, file.getContent().getSize(), file.getOwner() ); }catch (StorageIdNotFoundException e) { stringBuilder.append("\n").append("not found content id ").append(mongoId); throw e; @@ -161,7 +161,7 @@ public class MongoToMinioPortingNotShared implements AbstractScript{ stringBuilder.append("\n").append("searching on version for content id ").append(versionMongoId); try (InputStream stream = mongo.download(versionMongoId)){ - minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size); + minio.upload(stream, file.getContent().getRemotePath() , item.getName(), versionMongoId, size , file.getOwner()); } stringBuilder.append("\n").append("version copied ").append(name).append(" - ").append(file.getPath()); }