diff --git a/pom.xml b/pom.xml index f728b39..cad98ea 100644 --- a/pom.xml +++ b/pom.xml @@ -4,17 +4,17 @@ org.gcube.tools maven-parent - LATEST + 1.1.0 - org.gcube.spatial-data + org.gcube.spatial.data ws-thredds - 0.2.0-SNAPSHOT + 0.2.2-SNAPSHOT ws-thredds prototype of WS integration with data-transfer for Thredds pubblication ${project.basedir}/distro - http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId} + http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId} @@ -24,7 +24,17 @@ ${svnBaseUrl}/${project.artifactId} - + + + + org.gcube.distribution + maven-portal-bom + 3.4.0-SNAPSHOT + pom + import + + + @@ -32,38 +42,38 @@ org.gcube.spatial.data sdi-library [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + com.fasterxml.jackson.core + jackson-annotations + + - + org.gcube.data.transfer data-transfer-library [1.2.0-SNAPSHOT,2.0.0-SNAPSHOT) - - - - - - - org.gcube.common - home-library-jcr - [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) - + + + + org.gcube.common - home-library - [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) + storagehub-client-library - + org.slf4j - slf4j-log4j12 - 1.6.4 - provided + slf4j-log4j12 - - + + \ No newline at end of file diff --git a/requests.txt b/requests.txt deleted file mode 100644 index e69de29..0000000 diff --git a/src/main/java/org/gcube/usecases/ws/thredds/Commons.java b/src/main/java/org/gcube/usecases/ws/thredds/Commons.java index 28dcbb9..c2d0601 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/Commons.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/Commons.java @@ -14,7 +14,7 @@ import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; -import org.apache.tika.io.IOUtils; +import org.apache.commons.io.IOUtils; import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.client.AuthorizationFilter; diff --git a/src/main/java/org/gcube/usecases/ws/thredds/Constants.java b/src/main/java/org/gcube/usecases/ws/thredds/Constants.java index 3b81177..6e5df5e 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/Constants.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/Constants.java @@ -55,11 +55,11 @@ public class Constants { public static final String TRANSFERS_POOL_IDLE_MS="transfers.pool.idle.ms"; } - public static final Map cleanedItemPropertiesMap=new HashMap(); + public static final Map cleanedItemPropertiesMap=new HashMap(); - public static final Map cleanedFolderPropertiesMap=new HashMap(); + public static final Map cleanedFolderPropertiesMap=new HashMap(); - public static final Map defaultConfigurationMap=new HashMap(); + public static final Map defaultConfigurationMap=new HashMap(); static { cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.TBS, null); diff --git a/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java b/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java index 10a2378..34107af 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java @@ -1,5 +1,7 @@ package org.gcube.usecases.ws.thredds; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import lombok.Synchronized; @@ -10,7 +12,7 @@ public class LocalConfiguration { private static LocalConfiguration instance=null; @Synchronized - private static final LocalConfiguration get() { + public static final LocalConfiguration get() { if(instance==null) instance=new LocalConfiguration(); return instance; @@ -21,16 +23,22 @@ public class LocalConfiguration { return (String) get().props.getOrDefault(property, Constants.defaultConfigurationMap.get(property)); }catch(Throwable t) { log.warn("Unable to get configuration property "+property,t); - return Constants.defaultConfigurationMap.get(property); + return Constants.defaultConfigurationMap.get(property)+""; } } + public Map asMap(){ + HashMap toReturn=new HashMap<>(); + for(Object key :props.keySet()) + toReturn.put(key+"", (String)props.getOrDefault(key, Constants.defaultConfigurationMap.get(key))); + return toReturn; + } //***************** INSTANCE Properties props; - public LocalConfiguration() { + private LocalConfiguration() { props=new Properties(); try{ props.load(this.getClass().getResourceAsStream("configuration.properties")); diff --git a/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java b/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java index 9627bec..c347923 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java @@ -38,20 +38,19 @@ public class NetUtils { } - public static String resolveRedirects(String url) throws IOException{ + public static URL resolveRedirects(URL url) throws IOException{ log.debug("Resolving redirect for url {} ",url); - URL urlObj=new URL(url); - HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); int status=connection.getResponseCode(); if(status>=300&&status<400){ String newUrl=connection.getHeaderField("Location"); log.debug("Following redirect from {} to {} ",url,newUrl); - return resolveRedirects(newUrl); + return resolveRedirects(new URL(newUrl)); }else return url; } - public static File download(String url) throws MalformedURLException, IOException { - return toFile(new URL(resolveRedirects(url)).openStream()); + public static File download(URL url) throws MalformedURLException, IOException { + return toFile(resolveRedirects(url).openStream()); } } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java b/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java index 035d661..ce175e2 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java @@ -1,283 +1,258 @@ package org.gcube.usecases.ws.thredds; -import java.io.File; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WrongItemTypeException; -import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; -import org.gcube.usecases.ws.thredds.engine.PublishRequest; -import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode; -import org.gcube.usecases.ws.thredds.engine.PublishRequest.PublishItem; -import org.gcube.usecases.ws.thredds.engine.TransferRequestServer; -import org.gcube.usecases.ws.thredds.engine.TransferRequestServer.Report; - import lombok.extern.slf4j.Slf4j; @Slf4j public class PublishFolders { - private static String WS_USER="fabio.sinibaldi"; - - - public static void main(String[] args) { - TokenSetter.set("/d4science.research-infrastructures.eu"); - HashSet configs=new HashSet<>(); - - -// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS - // String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files - String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST - -// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET - - String publishingUserToken="***REMOVED***"; //fabio @NextNext - - - FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE"); - // folderConfig.setProvidedMetadata(true); - // folderConfig.setMetadataFolderId(metadataFolderID); - - - - - configs.add(folderConfig); - - - - - - - - TransferRequestServer server=new TransferRequestServer(); - for(FolderConfiguration entry:configs){ - try{ - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace(); - // FolderReport report=new FolderReport(entry); - log.info("Managing {} ",entry); - WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId()); - handleFolder(ws,entry,server,folder); - - - - }catch(WorkspaceException e){ - System.err.println("WORKSPACE EXC "); - e.printStackTrace(System.err); - }catch(HomeNotFoundException e){ - System.err.println("WORKSPACE EXC "); - e.printStackTrace(System.err); - }catch(InternalErrorException e){ - System.err.println("WORKSPACE EXC "); - e.printStackTrace(System.err); - }catch(UserNotFoundException e){ - System.err.println("WORKSPACE EXC "); - e.printStackTrace(System.err); - }catch(Exception e){ - System.err.println("UNEXPECTED EXC"); - e.printStackTrace(System.err); - } - } - System.out.println("Waiting for service.. "); - server.waitCompletion(); - - Report report=server.getReport(); - - File reportFile =report.toFile(folderConfig); - System.out.println("Report at "+reportFile.getAbsolutePath()); - - - } - - /** - * For *.nc | *.ncml - * if relatedMetadataFolder contains .xml use meta - * else ask DT to generate it - * - * - * @param wsFolder - * @param config - * @param server - * @return - * @throws WorkspaceException - * @throws InternalErrorException - * @throws ItemNotFoundException - * @throws HomeNotFoundException - * @throws UserNotFoundException - */ - - public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ - - log.info("Handling folder {} ",folder.getPath()); - - cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName()); - - log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath()); - publishFolder(ws,config,server,folder); - } - - - public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ - - - - - //Access folder - List folderItems=folder.getChildren(); - - if(config.isIncludeSubfolders()) { - log.info("Going through subfolders first....."); - for(WorkspaceItem item:folderItems) { - try { - if(item.isFolder()) { - FolderConfiguration subConfig=new FolderConfiguration(config); - subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName()); - publishFolder(ws,subConfig,server,(WorkspaceFolder) item); - log.debug("Subfolder {} successfully analyzed ",item.getPath()); - } - }catch(Exception e) { - try{ - log.warn("Unabel to check item {} ",item.getPath(),e); - }catch(InternalErrorException e1) { - log.warn("Unabel to check item and to get Path {} ",item,e); - } - } - } - } - - - - log.debug("Checking for ncml files .... "); - for(WorkspaceItem item : folderItems) { - try { - if(!item.isFolder()) { - String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); - if(prefix.equals(".ncml")) { - PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken()); - if(config.isProvidedMetadata()) { - String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; - File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); - if (meta!=null) req.setMetadata(meta); - } - // TODO NB Check for queue - server.put(req); - } - } - }catch(Exception e) { - try{ - log.warn("Unabel to check item {} ",item.getPath(),e); - }catch(InternalErrorException e1) { - log.warn("Unabel to check item and to get Path {} ",item,e); - } - } - } - - log.debug("Checking nc files.. "); - - for(WorkspaceItem item:folder.getChildren()){ - try { - if(!item.isFolder()) { - String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); - if(prefix.equals(".nc")){ - // NC - PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken()); - if(config.isProvidedMetadata()) { - String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; - File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); - if (meta!=null) req.setMetadata(meta); - } - server.put(req); - } - } - }catch(Exception e) { - try{ - log.warn("Unabel to check item {} ",item.getPath(),e); - }catch(InternalErrorException e1) { - log.warn("Unabel to check item and to get Path {} ",item,e); - } - } - } - folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+"")); - log.debug("Creating requests... "); - } - - private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException { - try{ - WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId); - if(found==null) throw new ItemNotFoundException("Found item was null"); - return NetUtils.toFile(((ExternalFile)found).getData()); - }catch(ItemNotFoundException e) { - return null; - } - } - - // Remotely deletes folders which has history operations : RENAMING, REMOVAL - private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException { - List history=folder.getAccounting(); - long lastUpdateTimeMillis=getLastUpdateTime(folder); - Date lastUpdate=new Date(lastUpdateTimeMillis); - log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate)); - - - //look into history - boolean toDeleteCurrentFolder=false; - for(AccountingEntry entry: history) { - Date eventTime=entry.getDate().getTime(); - - switch(entry.getEntryType()) { - case REMOVAL: - - case RENAMING:{ - log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); - if(eventTime.after(lastUpdate)) { - log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); - toDeleteCurrentFolder=true; - } - } - } - if(toDeleteCurrentFolder) break; - } - - - //Delete Folder or scan children - if(toDeleteCurrentFolder) { - log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath); - try{ - Commons.cleanupFolder(remoteFolderPath,targetToken); - }catch(Throwable t) { - log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t); - if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish - throw t; - - } - }else { - log.info("Folder is not to be cleaned up. Checking children.."); - for(WorkspaceItem item:folder.getChildren()) - if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName()); - } - } - - private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException { - try{ - return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME)); - }catch(Throwable e) { - log.debug("Unable to get last update time for {}. Considering 0..",item.getPath()); - return 0l; - } - } - +// private static String WS_USER="fabio.sinibaldi"; +// +// +// public static void main(String[] args) { +// TokenSetter.set("/d4science.research-infrastructures.eu"); +// HashSet configs=new HashSet<>(); +// +// +//// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS +// // String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files +// String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST +// +//// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET +// +// String publishingUserToken="***REMOVED***"; //fabio @NextNext +// +// +// FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE"); +// // folderConfig.setProvidedMetadata(true); +// // folderConfig.setMetadataFolderId(metadataFolderID); +// +// +// +// +// configs.add(folderConfig); +// +// +// +// +// +// +// +// TransferRequestServer server=new TransferRequestServer(); +// for(FolderConfiguration entry:configs){ +// try{ +// Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace(); +// // FolderReport report=new FolderReport(entry); +// log.info("Managing {} ",entry); +// WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId()); +// handleFolder(ws,entry,server,folder); +// +// +// +// }catch(WorkspaceException e){ +// System.err.println("WORKSPACE EXC "); +// e.printStackTrace(System.err); +// }catch(HomeNotFoundException e){ +// System.err.println("WORKSPACE EXC "); +// e.printStackTrace(System.err); +// }catch(InternalErrorException e){ +// System.err.println("WORKSPACE EXC "); +// e.printStackTrace(System.err); +// }catch(UserNotFoundException e){ +// System.err.println("WORKSPACE EXC "); +// e.printStackTrace(System.err); +// }catch(Exception e){ +// System.err.println("UNEXPECTED EXC"); +// e.printStackTrace(System.err); +// } +// } +// System.out.println("Waiting for service.. "); +// server.waitCompletion(); +// +// Report report=server.getReport(); +// +// File reportFile =report.toFile(folderConfig); +// System.out.println("Report at "+reportFile.getAbsolutePath()); +// +// +// } +// +// /** +// * For *.nc | *.ncml +// * if relatedMetadataFolder contains .xml use meta +// * else ask DT to generate it +// * +// * +// * @param wsFolder +// * @param config +// * @param server +// * @return +// * @throws WorkspaceException +// * @throws InternalErrorException +// * @throws ItemNotFoundException +// * @throws HomeNotFoundException +// * @throws UserNotFoundException +// */ +// +// public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ +// +// log.info("Handling folder {} ",folder.getPath()); +// +// cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName()); +// +// log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath()); +// publishFolder(ws,config,server,folder); +// } +// +// +// public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ +// +// +// +// +// //Access folder +// List folderItems=folder.getChildren(); +// +// if(config.isIncludeSubfolders()) { +// log.info("Going through subfolders first....."); +// for(WorkspaceItem item:folderItems) { +// try { +// if(item.isFolder()) { +// FolderConfiguration subConfig=new FolderConfiguration(config); +// subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName()); +// publishFolder(ws,subConfig,server,(WorkspaceFolder) item); +// log.debug("Subfolder {} successfully analyzed ",item.getPath()); +// } +// }catch(Exception e) { +// try{ +// log.warn("Unabel to check item {} ",item.getPath(),e); +// }catch(InternalErrorException e1) { +// log.warn("Unabel to check item and to get Path {} ",item,e); +// } +// } +// } +// } +// +// +// +// log.debug("Checking for ncml files .... "); +// for(WorkspaceItem item : folderItems) { +// try { +// if(!item.isFolder()) { +// String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); +// if(prefix.equals(".ncml")) { +// PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken()); +// if(config.isProvidedMetadata()) { +// String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; +// File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); +// if (meta!=null) req.setMetadata(meta); +// } +// // TODO NB Check for queue +// server.put(req); +// } +// } +// }catch(Exception e) { +// try{ +// log.warn("Unabel to check item {} ",item.getPath(),e); +// }catch(InternalErrorException e1) { +// log.warn("Unabel to check item and to get Path {} ",item,e); +// } +// } +// } +// +// log.debug("Checking nc files.. "); +// +// for(WorkspaceItem item:folder.getChildren()){ +// try { +// if(!item.isFolder()) { +// String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); +// if(prefix.equals(".nc")){ +// // NC +// PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken()); +// if(config.isProvidedMetadata()) { +// String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; +// File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); +// if (meta!=null) req.setMetadata(meta); +// } +// server.put(req); +// } +// } +// }catch(Exception e) { +// try{ +// log.warn("Unabel to check item {} ",item.getPath(),e); +// }catch(InternalErrorException e1) { +// log.warn("Unabel to check item and to get Path {} ",item,e); +// } +// } +// } +// folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+"")); +// log.debug("Creating requests... "); +// } +// +// private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException { +// try{ +// WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId); +// if(found==null) throw new ItemNotFoundException("Found item was null"); +// return NetUtils.toFile(((ExternalFile)found).getData()); +// }catch(ItemNotFoundException e) { +// return null; +// } +// } +// +// // Remotely deletes folders which has history operations : RENAMING, REMOVAL +// private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException { +// List history=folder.getAccounting(); +// long lastUpdateTimeMillis=getLastUpdateTime(folder); +// Date lastUpdate=new Date(lastUpdateTimeMillis); +// log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate)); +// +// +// //look into history +// boolean toDeleteCurrentFolder=false; +// for(AccountingEntry entry: history) { +// Date eventTime=entry.getDate().getTime(); +// +// switch(entry.getEntryType()) { +// case REMOVAL: +// +// case RENAMING:{ +// log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); +// if(eventTime.after(lastUpdate)) { +// log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); +// toDeleteCurrentFolder=true; +// } +// } +// } +// if(toDeleteCurrentFolder) break; +// } +// +// +// //Delete Folder or scan children +// if(toDeleteCurrentFolder) { +// log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath); +// try{ +// Commons.cleanupFolder(remoteFolderPath,targetToken); +// }catch(Throwable t) { +// log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t); +// if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish +// throw t; +// +// } +// }else { +// log.info("Folder is not to be cleaned up. Checking children.."); +// for(WorkspaceItem item:folder.getChildren()) +// if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName()); +// } +// } +// +// private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException { +// try{ +// return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME)); +// }catch(Throwable e) { +// log.debug("Unable to get last update time for {}. Considering 0..",item.getPath()); +// return 0l; +// } +// } +// } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java b/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java index b8089d8..bb8b92e 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java @@ -9,6 +9,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.SynchEngineImpl; import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException; import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor; import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor; import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; @@ -52,6 +53,6 @@ public interface SyncEngine { public Set getAvailableCatalogsByToken(String token) throws InternalException; - + public SyncEngineStatusDescriptor getStatus(); } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java index a740a09..4cfb568 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java @@ -3,9 +3,6 @@ package org.gcube.usecases.ws.thredds.engine; import java.io.File; import java.util.HashSet; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; @@ -23,11 +20,14 @@ public class PublishRequest { String name; String id; - public PublishItem(WorkspaceItem item) throws InternalErrorException { - url=item.getPublicLink(false); - name=item.getName(); - id=item.getId(); - } +// public PublishItem(Item item) throws InternalErrorException { +// +// +// +// url=item.getPublicLink(false); +// name=item.getName(); +// id=item.getId(); +// } } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java index b9a53fe..01f020b 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java @@ -1,17 +1,14 @@ package org.gcube.usecases.ws.thredds.engine; -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.net.URL; import java.nio.file.Files; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.library.faults.DestinationNotSetException; @@ -25,13 +22,7 @@ import org.gcube.data.transfer.library.faults.UnreachableNodeException; import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.data.transfer.model.PluginInvocation; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -import org.gcube.spatial.data.sdi.interfaces.Metadata; -import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions; import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; -import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder; -import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; import org.gcube.spatial.data.sdi.utils.ScopeUtils; import org.gcube.usecases.ws.thredds.Commons; import org.gcube.usecases.ws.thredds.NetUtils; @@ -90,7 +81,7 @@ public class PublishThread implements Runnable { waitFor(request.getQueueId(), request.getQueueCount()); log.debug("Loading netcdfFile .."); - File ncmlFile=NetUtils.download(request.getSource().getUrl()); + File ncmlFile=NetUtils.download(new URL(request.getSource().getUrl())); String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath())); for(String reportId:request.getToGatherReportsId()) { @@ -209,7 +200,7 @@ public class PublishThread implements Runnable { log.debug("Alerting queue {}. Expected count is {} ",queueId,expected); Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1)); sem.release(); - log.debug("Queue {} alerted. Remaining : {} out of {} ",queueId,sem.availablePermits(),expected); + log.debug(String.format("Queue %1$s alerted. Remaining : %2$s out of %3$s ",queueId,sem.availablePermits(),expected)); } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java index 81d1cfb..d3134d6 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java @@ -11,9 +11,10 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status; @@ -22,6 +23,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread; import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest; import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferToThreddsRequest; import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException; import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; @@ -37,7 +39,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class Process { - + private boolean submittedRequests=false; private ProcessDescriptor descriptor; @@ -48,7 +50,7 @@ public class Process { private Queue queuedReports=new LinkedList<>(); -// private String folderId; + // private String folderId; private WorkspaceFolderManager manager; @@ -58,13 +60,13 @@ public class Process { public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException { log.debug("Created Process with id {} ",processId); -// this.folderId=folderId; + // this.folderId=folderId; manager=new WorkspaceFolderManager(folderId); manager.lock(processId); SynchFolderConfiguration folderConfig=manager.getSynchConfiguration(); try { - descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().getPath(),System.currentTimeMillis(),processId,folderConfig); + descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().get().getPath(),System.currentTimeMillis(),processId,folderConfig); }catch(Exception e) { throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e); } @@ -75,10 +77,10 @@ public class Process { } - public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalErrorException, InternalException { + public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, StorageHubException, InternalException { WorkspaceUtils.resetStatus(manager.getTheFolder()); status.setCurrentMessage("Analyzing folder.."); - + generateRequests(this,service, manager.getTheFolder()); submittedRequests=true; if(status.getQueuedTransfers().get()>0) { @@ -164,6 +166,7 @@ public class Process { break; } } + status.setCurrentMessage("Waiting for requests ["+(status.getErrorCount().get()+status.getServedTransfers().get())+"/"+status.getQueuedTransfers().get()+"] to be served."); status.getLogBuilder().append( String.format("%s - item [%s] %s: %s \n", Constants.DATE_FORMAT.format(new Date(report.getCompletionTime())), report.getElementName(),report.getStatus()+"",report.getMessage())); @@ -203,27 +206,27 @@ public class Process { } - private static final void generateRequests(Process ownerProcess,ExecutorService service,WorkspaceFolder toScanFolder ) throws InternalErrorException, InternalException{ - - String folderPath=toScanFolder.getPath(); - + private static final void generateRequests(Process ownerProcess,ExecutorService service,FolderContainer toScanFolder ) throws StorageHubException, InternalException{ + + String folderPath=toScanFolder.get().getPath(); + log.info("Generating requests for folder {}",folderPath); log.debug("Process is {} ",ownerProcess.getDescriptor()); Set handledWorkspaceItemEntries=new HashSet(); - + SynchFolderConfiguration config=ownerProcess.getDescriptor().getSynchConfiguration(); Set remoteChildrenNames; Set localChildrenNames=new HashSet<>(); - - List localFolderChildren=toScanFolder.getChildren(); - for(WorkspaceItem item:localFolderChildren) { - localChildrenNames.add(item.getName()); + + List> localFolderChildren=toScanFolder.list().withAccounting().withMetadata().getContainers(); + for(ItemContainer item:localFolderChildren) { + localChildrenNames.add(item.get().getName()); } - - String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + + String relativePath=toScanFolder.get().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+""; ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken()); - + RemoteFileDescriptor folderDesc=null; try{ folderDesc=folderController.getFileDescriptor(); @@ -232,41 +235,41 @@ public class Process { folderController.createEmptyFolder(null); folderDesc=folderController.getFileDescriptor(); } - + remoteChildrenNames=new HashSet<>(folderDesc.getChildren()); - - + + //*********************** HANDLING ACCOUNTING ENTRIES - + Set handledAccountingEntries=WorkspaceUtils.scanAccountingForStatus( toScanFolder, config, localChildrenNames, remoteChildrenNames, folderController, ownerProcess, service); - - - + + + //SCAN FOLDER CONTENT log.debug("Checking content of {} ",folderPath); - for(WorkspaceItem item:localFolderChildren) { + for(ItemContainer item:localFolderChildren) { - if(item.isFolder()) { + if(item.getType().equals(ContainerType.FOLDER)) { // RECURSIVE ON SUB FOLDERS - generateRequests(ownerProcess,service,(WorkspaceFolder) item); + generateRequests(ownerProcess,service,(FolderContainer) item); }else { - Map props=item.getProperties().getProperties(); + Map props=item.get().getMetadata().getMap(); String itemId=item.getId(); - String itemName=item.getName(); + String itemName=item.get().getName(); // REQUESTS ARE EVALUATED ON PROPERTIES (SET BY PREVIOUS SCAN) - + if(props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null)) { try { - SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)); - log.trace("Found TBS item {}, name {}, status : ",item.getId(),item.getName(),status); + SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)+""); + log.trace(String.format("Found TBS item %1$s, name %2$s, status : %3$s",item.getId(),item.get().getName(),status)); SynchronizationRequest request=null; switch(status) { - case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder,item); + case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder.get(),item.get()); break; - case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item, toScanFolder, null); + case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item.get(), toScanFolder.get(), null); break; } @@ -277,34 +280,45 @@ public class Process { }else log.debug("Item is up to date"); handledWorkspaceItemEntries.add(itemName); }catch(Throwable t) { - log.error("Unable to submit request for {} ID {} ",itemName,itemId,t); + log.error(String.format("Unable to submit request for %1$s ID %2$s ",itemName,itemId),t); } } } } - + // check items to be imported - + try { Set toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, handledAccountingEntries, handledWorkspaceItemEntries, toScanFolder, folderController, config, ownerProcess, service); log.debug("Checking if remote location contains folders to be imported..."); - for(String item:toImportItems) { - if(folderController.getFileDescriptor(item).isDirectory()) { - log.info("Creating folder {} under {} ",item,folderPath); - try{ - WorkspaceFolder folder=toScanFolder.createFolder(item, "Imported from thredds"); + for(String item:toImportItems) { + if(folderController.getFileDescriptor(item).isDirectory()) { + + //check if folder already existing!! + + FolderContainer folder=null; + try{ + try { + ItemContainer foundElement=WorkspaceUtils.scan(toScanFolder, item); + if(!foundElement.getType().equals(ContainerType.FOLDER)) + throw new Exception("Conflict on WS item "+foundElement.getId()+". Remote element is a folder "); + }catch(ItemNotFoundException e) { + log.info("Creating folder {} under {} ",item,folderPath); + folder=toScanFolder.newFolder(item, "Imported from thredds"); WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId()); generateRequests(ownerProcess, service, folder); - }catch(Throwable t) { - log.error("Unable to import folder {} into {} ",item,folderPath); } + + }catch(Throwable t) { + log.error("Unable to import folder {} into {} ",item,folderPath); } } + } }catch(InternalException e) { log.error("Unable to check remote content with config {} ",config,e); } - - + + log.info("All requests for {} synchronization have been submitted [count {} ]. ",folderPath,ownerProcess.status.getQueuedTransfers().get()); } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java index 8095f97..86d371a 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java @@ -7,7 +7,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -27,6 +26,7 @@ import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException; import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; import org.gcube.usecases.ws.thredds.model.CompletionCallback; +import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor; import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor; import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; @@ -53,15 +53,20 @@ public class SynchEngineImpl implements SyncEngine{ localProcesses=new ConcurrentHashMap<>(); + // NB UNBOUNDED QUEUE MEANS ONLY CORE THREADS ARE EXECUTED + int scannerMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_MAX_SIZE)); int scannerCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_CORE_SIZE)); +// int scannerCoreSize=scannerMaxSize; int scannerIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_IDLE_MS)); int transfersMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_MAX_SIZE)); int transfersCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_CORE_SIZE)); +// int transfersCoreSize=transfersMaxSize; int transfersIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_IDLE_MS)); + initializationExecutor= new ThreadPoolExecutor(scannerCoreSize, scannerMaxSize, scannerIdleMs, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); @@ -264,4 +269,10 @@ public class SynchEngineImpl implements SyncEngine{ return toReturn; } + + @Override + public SyncEngineStatusDescriptor getStatus() { + ThreadPoolExecutor exec=(ThreadPoolExecutor) synchronizationExecutor; + return new SyncEngineStatusDescriptor(exec.getActiveCount(), exec.getQueue().size(), LocalConfiguration.get().asMap()); + } } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java index a218a0b..90189b9 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; +import java.net.URL; import java.nio.file.Files; import java.util.Set; @@ -16,7 +17,8 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.tika.io.IOUtils; +import org.apache.commons.io.IOUtils; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.TransferResult; @@ -83,10 +85,10 @@ public class ThreddsController { private void resetCallerToken() { if(callerToken!=null) { - log.trace("Resetting caller token {}. Target Token is {}, current is {} ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())); + log.trace(String.format("Resetting caller token %1$s. Target Token is %2$s, current is %3$s ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken()))); TokenSetter.setToken(callerToken); callerToken=null; - }else log.trace("Caller token {} already reset [current token {}]. Target Token is {}",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)); + }else log.trace(String.format("Caller token %1$s already reset [current token %2$s]. Target Token is %3$s",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken))); } public final ThreddsInfo getThreddsInfo() { @@ -208,6 +210,7 @@ public class ThreddsController { public ThreddsCatalog createCatalog(String name) throws InternalException { setTargetToken(); try{ + SecurityTokenProvider.instance.get(); log.info("Creating catalog with name {} for path {} ",name,operatingPath); String sdiUrl="http://"+getSDIServiceHost()+"/"+Constants.SDI_THREDDS_BASE_URL; Response resp=getWebClient().target(sdiUrl). @@ -267,7 +270,7 @@ public class ThreddsController { } - public TransferResult transferFile(Destination dest,String url,Set invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { + public TransferResult transferFile(Destination dest,URL url,Set invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { setTargetToken(); try{DataTransferClient client=getDTClient(hostname); if(invocations!=null&&!invocations.isEmpty()) diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java index e46cefc..dae4da4 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java @@ -1,20 +1,17 @@ package org.gcube.usecases.ws.thredds.engine.impl; import java.io.File; -import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.Metadata; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog; import org.gcube.usecases.ws.thredds.Constants; @@ -43,7 +40,7 @@ public class WorkspaceFolderManager { } - private WorkspaceFolder theFolder; + private FolderContainer theFolder; private String folderId; @@ -52,20 +49,21 @@ public class WorkspaceFolderManager { private SynchFolderConfiguration config=null; private ThreddsController threddsController=null; - private Workspace ws; + private StorageHubClient sc; public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException { try{ - ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - theFolder=(WorkspaceFolder) ws.getItem(folderId); +// ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + sc=WorkspaceUtils.getClient(); + theFolder=sc.open(folderId).asFolder(); this.folderId=folderId; - }catch(WorkspaceException | InternalErrorException | HomeNotFoundException | UserNotFoundException e) { + }catch(StorageHubException e) { throw new WorkspaceInteractionException("Unable to access folder id "+folderId,e); } } - public WorkspaceFolder getTheFolder() { + public FolderContainer getTheFolder() { return theFolder; } @@ -79,21 +77,17 @@ public class WorkspaceFolderManager { private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { try { - WorkspaceFolder root=(WorkspaceFolder) ws.getItem(getSynchConfiguration().getRootFolderId()); + FolderContainer root=sc.open(getSynchConfiguration().getRootFolderId()).asFolder(); SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root); return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken()); - }catch(WorkspaceException | InternalErrorException e) { + }catch(StorageHubException e) { throw new WorkspaceInteractionException(e); } } public boolean isRoot() throws WorkspaceNotSynchedException, WorkspaceInteractionException{ - try{ - return getSynchConfiguration().getRootFolderId().equals(theFolder.getId()); - }catch(InternalErrorException e) { - throw new WorkspaceInteractionException(e); - } + return getSynchConfiguration().getRootFolderId().equals(theFolder.getId()); } @@ -103,20 +97,16 @@ public class WorkspaceFolderManager { if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched."); log.debug("Loading properties for "); config=WorkspaceUtils.loadConfiguration(theFolder); - }catch(InternalErrorException e) { + }catch(StorageHubException e) { throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e); } } return config; } - public boolean isSynched() throws WorkspaceInteractionException { - try{ - Map props=theFolder.getProperties().getProperties(); + public boolean isSynched(){ + Map props=theFolder.get().getMetadata().getMap(); return props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null); - }catch(InternalErrorException e) { - throw new WorkspaceInteractionException("Unable to check Synch flag on "+folderId,e); - } } @@ -126,9 +116,9 @@ public class WorkspaceFolderManager { SynchFolderConfiguration config=getSynchConfiguration(); try{ - checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder)); - return new SyncFolderDescriptor(this.folderId,this.theFolder.getPath(),config); - }catch(InternalErrorException e) { + checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder.get())); + return new SyncFolderDescriptor(this.folderId,this.theFolder.get().getPath(),config); + }catch(StorageHubException e) { throw new WorkspaceInteractionException(e); } } @@ -188,7 +178,7 @@ public class WorkspaceFolderManager { }catch(InternalException e) { throw new InternalException ("Unable to check/initialize remote folder",e); - }catch(InternalErrorException e) { + }catch(StorageHubException e) { throw new WorkspaceInteractionException("Unable to set Properties to "+folderId,e); } } @@ -202,12 +192,12 @@ public class WorkspaceFolderManager { WorkspaceUtils.cleanItem(theFolder); if(deleteRemote) getThreddsController().createEmptyFolder(null); - }catch(InternalErrorException e) { + }catch(StorageHubException e) { throw new WorkspaceInteractionException("Unable to cleanup "+folderId,e); } } - public void setLastUpdateTime() throws InternalErrorException { + public void setLastUpdateTime() throws StorageHubException { WorkspaceUtils.setLastUpdateTime(theFolder, System.currentTimeMillis()); } @@ -251,12 +241,12 @@ public class WorkspaceFolderManager { - private static void checkFolder(WorkspaceFolder folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws InternalErrorException, InternalException { + private static void checkFolder(FolderContainer folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws StorageHubException, InternalException { // Check folder configuration - log.trace("Checking folder {} ",folder.getPath()); + log.trace("Checking folder {} ",folder.get().getPath()); log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder); - String folderName=folder.getName(); + String folderName=folder.get().getName(); String currentRemotePath=rootConfig.getRemotePath()+((relativePathFromRootFolder==null)?"":"/"+relativePathFromRootFolder); @@ -270,19 +260,19 @@ public class WorkspaceFolderManager { log.debug("Initializing properties for {} ",folderName); //INIT PROPERTIES IF NOT PRESENT - if(!WorkspaceUtils.isConfigured(folder)) + if(!WorkspaceUtils.isConfigured(folder.get())) WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId); - for(WorkspaceItem item:folder.getChildren()) { - String itemName=item.getName(); + for(ItemContainer item:folder.list().withAccounting().withMetadata().getContainers()) { + String itemName=item.get().getName(); String itemRelativePath=(relativePathFromRootFolder==null)?itemName:relativePathFromRootFolder+"/"+itemName; String itemRemotePath=currentRemotePath+"/"+itemName; - if(item.isFolder()) { + if(item.getType().equals(ContainerType.FOLDER)) { if(recursive) - checkFolder((WorkspaceFolder) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine); + checkFolder((FolderContainer) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine); else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId); }else if(rootConfig.matchesFilter(itemName)) { - if(!WorkspaceUtils.isConfigured(item)) + if(!WorkspaceUtils.isConfigured(item.get())) WorkspaceUtils.initProperties(item, null, null, null,null,null,null); } currentFolderExistingItem.add(itemName); @@ -305,31 +295,50 @@ public class WorkspaceFolderManager { currentFolderExistingItem, remoteFolderItems, controller, null, null); if(accountingEntries.isEmpty()) { - log.debug("No accounting entries found"); + log.debug("No Accounting Entries to be managed.."); folderStatus=SynchronizationStatus.UP_TO_DATE; } // CHECK WS ITEMS - for(WorkspaceItem item:folder.getChildren()) - if(item.isFolder()||rootConfig.matchesFilter(item.getName())) { - SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item, remoteFolderItems, controller,lastUpdatedRoutine); - item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+"")); + for(ItemContainer item:folder.list().withAccounting().withMetadata().getContainers()) + if(item.getType().equals(ContainerType.FOLDER)||rootConfig.matchesFilter(item.get().getName())) { + SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item.get(), remoteFolderItems, controller,lastUpdatedRoutine); + + Metadata meta=item.get().getMetadata(); + Map map=meta.getMap(); + map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+""); + meta.setMap(map); + item.setMetadata(meta); folderStatus=folderStatus.equals(SynchronizationStatus.UP_TO_DATE)?itemStatus:folderStatus; - } + } // CHECK REMOTE FOLDER if(folderStatus.equals(SynchronizationStatus.UP_TO_DATE)) { Set toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, accountingEntries, currentFolderExistingItem, folder, controller, rootConfig, null, null); if(!toImportItems.isEmpty()) folderStatus=SynchronizationStatus.OUTDATED_WS; } - - folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+"")); + + Metadata meta=folder.get().getMetadata(); + Map map=meta.getMap(); + map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+""); + meta.setMap(map); + folder.setMetadata(meta); }else { // Remote Folder not existing, set everything to OUTDATED_REMOTE - for(WorkspaceItem item:folder.getChildren()) - item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"")); - folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"")); + for(ItemContainer item:folder.list().withMetadata().getContainers()) { + Metadata meta=item.get().getMetadata(); + Map map=meta.getMap(); + map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""); + meta.setMap(map); + item.setMetadata(meta); + } + + Metadata meta=folder.get().getMetadata(); + Map map=meta.getMap(); + map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""); + meta.setMap(map); + folder.setMetadata(meta); } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java index 85bd453..f255150 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java @@ -1,26 +1,31 @@ package org.gcube.usecases.ws.thredds.engine.impl; -import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; -import org.gcube.common.homelibary.model.items.type.WorkspaceItemType; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.Properties; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; -import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRemoval; -import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRenaming; + +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.Metadata; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; +import org.gcube.common.storagehub.model.items.nodes.accounting.AccountEntry; +import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRemoval; +import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRenaming; +import org.gcube.common.storagehub.model.types.WorkspaceItemType; import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest; import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread; import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest; import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException; import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; import org.gcube.usecases.ws.thredds.model.StepReport; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; @@ -33,10 +38,12 @@ public class WorkspaceUtils { - + public static StorageHubClient getClient() throws StorageHubException{ + return new StorageHubClient(); + } /** - * Checks current accounting info in order to infere synchronization status. + * Checks current accounting info in order to infer synchronization status. * OwnerProcess and service parameters can be null for check purposes. * * @param folderPath @@ -52,53 +59,62 @@ public class WorkspaceUtils { * @throws InternalErrorException */ static Set scanAccountingForStatus( - WorkspaceFolder toScanFolder, + FolderContainer toScanFolder, SynchFolderConfiguration config, Set localChildrenNames, Set remoteChildrenNames, ThreddsController folderController, Process ownerProcess, - ExecutorService service) throws InternalErrorException{ + ExecutorService service) throws StorageHubException{ Set handledAccountingEntries=new HashSet<>(); - log.debug("Checking history of {} ",toScanFolder.getPath()); + FolderItem toScanFolderItem=toScanFolder.get(); + + + log.debug("Checking history of {} ",toScanFolderItem.getPath()); + + - String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + String relativePath=toScanFolderItem.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+""; Date folderLastUpdateTime=null; try{ - folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolder); + folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolderItem); }catch(Throwable t) { - log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolder.getName(),t); + log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolderItem.getName(),t); folderLastUpdateTime=new Date(0l); } // scanning for deletions - log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolder.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime)); - for(AccountingEntry entry:toScanFolder.getAccounting()) { + log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolderItem.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime)); + for(AccountEntry entry:toScanFolderItem.getAccounting().getEntries()) { try { Date eventTime=entry.getDate().getTime(); if(folderLastUpdateTime==null|| eventTime.after(folderLastUpdateTime)) { // SKIP IF ENTRY OLDER THAN LAST UPDATE TIME String toDeleteRemote=null; - switch(entry.getEntryType()) { + switch(entry.getType()) { case CUT: case REMOVAL:{ - AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry; - if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER)|| + AccountFolderEntryRemoval removalEntry=(AccountFolderEntryRemoval) entry; + if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER.toString())|| config.matchesFilter(removalEntry.getItemName())) toDeleteRemote=removalEntry.getItemName(); break; } - case RENAMING:{ - AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry; - WorkspaceItem newItem=toScanFolder.find(renamingEntry.getNewItemName()); - if(newItem.isFolder()||config.matchesFilter(renamingEntry.getOldItemName())) + case RENAMING:{ + AccountFolderEntryRenaming renamingEntry=(AccountFolderEntryRenaming) entry; + ItemContainer newItem=toScanFolder.findByName(renamingEntry.getNewItemName()).getContainers().get(0); + if(newItem.getType().equals(ContainerType.FOLDER)||config.matchesFilter(renamingEntry.getOldItemName())) toDeleteRemote=renamingEntry.getOldItemName(); break; - } + } + + default : { + log.debug("Skpping accounting entry {} ",entry.getType()); + } } @@ -117,7 +133,7 @@ public class WorkspaceUtils { if(service!=null) { log.debug("Service is not null. Submitting request ... "); if(eventTime.after(remoteDate)) { - service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolder,toDeleteRemote))); + service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolderItem,toDeleteRemote))); handledAccountingEntries.add(toDeleteRemote); log.debug("Submitted DELETION request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet()); } @@ -161,13 +177,15 @@ public class WorkspaceUtils { RemoteFileDescriptor folderDesc, Set handledAccountingEntries, Set handledWorkspaceItemEntries, - WorkspaceFolder toScanFolder, + FolderContainer toScanFolder, ThreddsController folderController, SynchFolderConfiguration config, Process ownerProcess, - ExecutorService service) throws InternalException, InternalErrorException{ + ExecutorService service) throws InternalException, StorageHubException{ - log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanFolder.getPath(),folderDesc.getAbsolutePath()); + FolderItem toScanItem=toScanFolder.get(); + + log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanItem.getPath(),folderDesc.getAbsolutePath()); Set handledRemoteElements=new HashSet(); // String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); @@ -189,7 +207,7 @@ public class WorkspaceUtils { log.debug("Child {} matches filter..."); handledRemoteElements.add(child); if(service!=null) { - service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, child))); + service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanItem, child))); log.debug("Submitted IMPORT request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet()); } // import if matching @@ -201,10 +219,12 @@ public class WorkspaceUtils { } - static void initProperties(WorkspaceItem toInit, String remotePath, String filter, String targetToken, - String catalogName,Boolean validateMeta, String rootFolderId) throws InternalErrorException { + static void initProperties(ItemContainer toInit, String remotePath, String filter, String targetToken, + String catalogName,Boolean validateMeta, String rootFolderId) throws StorageHubException { - Map toSetProperties=toInit.getProperties().getProperties(); + + Metadata meta=toInit.get().getMetadata(); + Map toSetProperties=meta.getMap(); initIfMissing(toSetProperties,Constants.WorkspaceProperties.TBS,"true"); @@ -212,7 +232,7 @@ public class WorkspaceUtils { initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_STATUS,StepReport.Status.OK+""); initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.UP_TO_DATE+""); - if(toInit.isFolder()) { + if(toInit.getType().equals(ContainerType.FOLDER)) { initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter); initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath); initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE); @@ -222,26 +242,28 @@ public class WorkspaceUtils { initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId); }else { initIfMissing(toSetProperties,Constants.WorkspaceProperties.METADATA_UUID,null); - } - toInit.getProperties().addProperties(toSetProperties); + } + meta.setMap(toSetProperties); + toInit.setMetadata(meta); + } - private static void initIfMissing(Map current,String key,String defaultValue) { + private static void initIfMissing(Map current,String key,String defaultValue) { if(!current.containsKey(key)|| current.get(key)==null|| current.get(key).equals("null")) current.put(key, defaultValue); } - static boolean isConfigured(WorkspaceItem toCheck) throws InternalErrorException { - return isConfigured(toCheck.getProperties().getProperties()); + static boolean isConfigured(Item toCheck) throws StorageHubException { + return isConfigured(toCheck.getMetadata().getMap()); } - static boolean isConfigured(Map toCheckProperties) { + static boolean isConfigured(Map toCheckProperties) { return (toCheckProperties.containsKey(Constants.WorkspaceProperties.TBS)&&toCheckProperties.get(Constants.WorkspaceProperties.TBS)!=null); } - static SynchronizationStatus getStatusAgainstRemote(WorkspaceItem item, Set existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException, InternalErrorException, RemoteFileNotFoundException { + static SynchronizationStatus getStatusAgainstRemote(Item item, Set existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException,RemoteFileNotFoundException { String itemName=item.getName(); SynchronizationStatus status=SynchronizationStatus.OUTDATED_REMOTE; if(existingRemote.contains(itemName)) { @@ -269,88 +291,132 @@ public class WorkspaceUtils { } - // - // /** - // * - // * @return max date between creation time, last modification time && LAST-UPDATE-PROP - // * @throws InternalErrorException - // * @throws NumberFormatException - // */ - // static Date getMaxLastUpdate(WorkspaceItem item) throws NumberFormatException, InternalErrorException { - // return new Date(Long.max(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)),item.getLastModificationTime().getTimeInMillis())); - // } - static Date safelyGetLastUpdate(WorkspaceItem item) throws InternalErrorException { + static Date safelyGetLastUpdate(Item item){ try { - return new Date(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME))); + return new Date(Long.parseLong(item.getMetadata().getMap().get(Constants.WorkspaceProperties.LAST_UPDATE_TIME)+"")); }catch(NumberFormatException e) { log.debug("Unable to get last update time for {} ",item.getName(),e); return new Date(0l); } } - public static boolean isModifiedAfter(WorkspaceItem item,Date fromDate) throws InternalErrorException { - for(AccountingEntry entry:item.getAccounting()) { - if(entry.getDate().getTime().after(fromDate)) { - switch(entry.getEntryType()) { - case PASTE: - case CREATE: - case RESTORE: - case UPDATE: - case ADD: return true; - } - } - } - return false; + public static boolean isModifiedAfter(Item item,Date fromDate) { + return item.getLastModificationTime().after(fromDate); +// for(AccountingEntry entry:item.getAccounting()) { +// if(entry.getDate().getTime().after(fromDate)) { +// switch(entry.getEntryType()) { +// case PASTE: +// case CREATE: +// case RESTORE: +// case UPDATE: +// case ADD: return true; +// } +// } +// } +// return false; } - static void cleanItem(WorkspaceItem item) throws InternalErrorException { - Properties props=item.getProperties(); - if(props.hasProperty(Constants.WorkspaceProperties.TBS)) { - if(item.isFolder()) { - props.addProperties(Constants.cleanedFolderPropertiesMap); - for(WorkspaceItem child : ((WorkspaceFolder)item).getChildren()) + static void cleanItem(ItemContainer itemContainer) throws StorageHubException{ + Metadata meta=itemContainer.get().getMetadata(); + Map map=meta.getMap(); + if(map.containsKey(Constants.WorkspaceProperties.TBS)) { + if(itemContainer.getType().equals(ContainerType.FOLDER)) { + map=Constants.cleanedFolderPropertiesMap; + for(ItemContainer child : ((FolderContainer)itemContainer).list().withMetadata().getContainers()) cleanItem(child); - }else props.addProperties(Constants.cleanedItemPropertiesMap); + }else map=Constants.cleanedItemPropertiesMap; + + //Actually posting cleaned map + meta.setMap(map); + log.debug("Setting meta to item "+itemContainer.getId()+" : "+itemContainer.get().getPath()); + itemContainer.setMetadata(meta); } } - static void setLastUpdateTime(WorkspaceFolder folder,long toSetTime) throws InternalErrorException { - StepReport.Status currentWSStatus=StepReport.Status.valueOf(folder.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)); + static void setLastUpdateTime(FolderContainer folder,long toSetTime) throws StorageHubException { + + Metadata meta=folder.get().getMetadata(); + Map map=meta.getMap(); + + StepReport.Status currentWSStatus=StepReport.Status.valueOf(map.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+""); if(currentWSStatus.equals(StepReport.Status.OK)) - folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+"")); + map.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+""); - for(WorkspaceItem item:folder.getChildren()) - if(item.isFolder()) setLastUpdateTime((WorkspaceFolder) item, toSetTime); + //Actually posting cleaned map + meta.setMap(map); + folder.setMetadata(meta); + +// for(ItemContainer child : folder.list().withMetadata().getContainers()) +// if(child.getType().equals(ContainerType.FOLDER)) +// setLastUpdateTime((FolderContainer) child, toSetTime); } - public static SynchFolderConfiguration loadConfiguration(WorkspaceItem item) throws InternalErrorException { - if(item.isFolder()) { - Properties props=item.getProperties(); + public static SynchFolderConfiguration loadConfiguration(ItemContainer item) throws StorageHubException { + if(item.getType().equals(ContainerType.FOLDER)) { + Map map=item.get().getMetadata().getMap(); SynchFolderConfiguration config=new SynchFolderConfiguration(); - config.setFilter(props.getPropertyValue(Constants.WorkspaceProperties.SYNCH_FILTER)); - config.setRemotePath(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH)); - config.setRemotePersistence(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PERSISTENCE)); - config.setTargetToken(props.getPropertyValue(Constants.WorkspaceProperties.TARGET_TOKEN)); - config.setToCreateCatalogName(props.getPropertyValue(Constants.WorkspaceProperties.RELATED_CATALOG)); - config.setValidateMetadata(Boolean.parseBoolean(props.getPropertyValue(Constants.WorkspaceProperties.VALIDATE_METADATA))); - config.setRootFolderId(props.getPropertyValue(Constants.WorkspaceProperties.ROOT_FOLDER_ID)); + config.setFilter(""+map.get(Constants.WorkspaceProperties.SYNCH_FILTER)); + config.setRemotePath(""+map.get(Constants.WorkspaceProperties.REMOTE_PATH)); + config.setRemotePersistence(""+map.get(Constants.WorkspaceProperties.REMOTE_PERSISTENCE)); + config.setTargetToken(""+map.get(Constants.WorkspaceProperties.TARGET_TOKEN)); + config.setToCreateCatalogName(""+map.get(Constants.WorkspaceProperties.RELATED_CATALOG)); + config.setValidateMetadata(Boolean.parseBoolean(""+map.get(Constants.WorkspaceProperties.VALIDATE_METADATA))); + config.setRootFolderId(""+map.get(Constants.WorkspaceProperties.ROOT_FOLDER_ID)); return config; - }else return loadConfiguration(item.getParent()); + }else { + FolderContainer parentFolder=getClient().open(item.get().getParentId()).asFolder(); + + return loadConfiguration(parentFolder); + } } - static void resetStatus(WorkspaceItem item) throws InternalErrorException { - if(item.isFolder()) { - for(WorkspaceItem child: ((WorkspaceFolder)item).getChildren()) + static void resetStatus(ItemContainer item) throws StorageHubException { + if(item.getType().equals(ContainerType.FOLDER)) { + for(ItemContainer child : ((FolderContainer)item).list().withMetadata().getContainers()) resetStatus(child); } - Map props=item.getProperties().getProperties(); - if(props.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) { - props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+""); - item.getProperties().addProperties(props); + Metadata meta=item.get().getMetadata(); + Map map=meta.getMap(); + if(map.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) { + map.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+""); + //Actually posting cleaned map + meta.setMap(map); + item.setMetadata(meta); } } + + public static void addParameters(ItemContainer item, Map toAdd) throws StorageHubException { + Metadata meta=item.get().getMetadata(); + Map current=meta.getMap(); + current.putAll(toAdd); + meta.setMap(current); + item.setMetadata(meta); + } + + public static void addParameter(ItemContainer item, String key, Object value) throws StorageHubException { + Metadata meta=item.get().getMetadata(); + Map current=meta.getMap(); + current.put(key,value); + meta.setMap(current); + item.setMetadata(meta); + } + + + public static ItemContainer scan(FolderContainer folder,String path) throws ItemNotFoundException, StorageHubException{ + String toLookFor=path.substring((path.startsWith("/")?1:0), path.length()); + String[] split=toLookFor.split("/"); + toLookFor=split[0]; + for(ItemContainer item:folder.list().withMetadata().getContainers()) + if(item.get().getName().equals(toLookFor)) { + if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length())); + else return item; + } + throw new ItemNotFoundException("Unable to find "+path); + } + + } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java index e82f29b..48952f5 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java @@ -1,6 +1,6 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.storagehub.model.items.FolderItem; import org.gcube.usecases.ws.thredds.engine.impl.Process; import lombok.Data; @@ -10,7 +10,7 @@ public class DeleteRemoteRequest extends SynchronizationRequest { private String toRemoveName; - public DeleteRemoteRequest(Process process,WorkspaceFolder location,String name) { + public DeleteRemoteRequest(Process process,FolderItem location,String name) { super(process,location); this.toRemoveName=name; } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java index bcb63a7..c0b5e1a 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java @@ -2,7 +2,7 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads; import java.util.concurrent.ExecutorService; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.usecases.ws.thredds.engine.impl.Process; import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager; @@ -45,7 +45,7 @@ public class ProcessInitializationThread implements Runnable { } catch (InternalException e) { log.error("Unable to proceed..",e); theProcess.cancel(); - } catch (InternalErrorException e) { + } catch (StorageHubException e) { log.error("Unable to proceed..",e); theProcess.cancel(); }catch(Throwable t) { diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java index f277ca0..31fcca6 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java @@ -3,10 +3,12 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads; import java.io.BufferedWriter; import java.io.FileWriter; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.SyncEngine; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; @@ -81,33 +83,30 @@ public class RequestLogger { } - private static final String getRemotePath(WorkspaceItem item) { + private static final String getRemotePath(Item item) { try{ - if(item.isFolder()) return item.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); - else return getRemotePath(item.getParent()); - }catch(InternalErrorException e) { + if(item instanceof FolderItem) return item.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+""; + else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get()); + }catch(StorageHubException e) { log.warn("Unable to get Remote Path ",e); return "N/A"; } } - private static final String getName(WorkspaceItem item) { - try { + private static final String getName(Item item) { return item.getName(); - }catch(InternalErrorException e) { - log.warn("Unable to get name ",e); - return "N/A"; - } } - private static final String getWSPath(WorkspaceItem item) { + private static final String getWSPath(Item item) { try{ - if(item.isFolder()) return item.getPath(); - else return getRemotePath(item.getParent()); - }catch(InternalErrorException e) { + if(item instanceof FolderItem) return item.getPath(); + else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get()); + }catch(StorageHubException e) { log.warn("Unable to get WS Path ",e); return "N/A"; } } + + } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java index 40c9d16..f3a95ca 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java @@ -1,6 +1,6 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.storagehub.model.items.FolderItem; import org.gcube.usecases.ws.thredds.engine.impl.Process; import lombok.Data; @@ -12,5 +12,5 @@ public abstract class SynchronizationRequest { @NonNull private Process process; @NonNull - private WorkspaceFolder location; + private FolderItem location; } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java index 8bd2dd1..d79f756 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java @@ -8,9 +8,15 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FileContainer; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.exceptions.ItemLockedException; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.DestinationClashPolicy; @@ -32,7 +38,6 @@ import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; import org.gcube.usecases.ws.thredds.faults.CancellationException; import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError; -import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; import org.gcube.usecases.ws.thredds.model.StepReport; import org.gcube.usecases.ws.thredds.model.StepReport.OperationType; @@ -60,22 +65,25 @@ public class SynchronizationThread implements Runnable { RequestLogger.get().log(theRequest); String reportMessage="Never started"; String reportItemName="Still Unknown"; - Status toSetStatus=Status.ERROR; - WorkspaceFolder parentFolder=theRequest.getLocation(); + Status toSetStatus=Status.ERROR; + + FolderItem parentFolderItem=theRequest.getLocation(); try { + StorageHubClient client=WorkspaceUtils.getClient(); + FolderContainer parentFolder=client.open(parentFolderItem.getId()).asFolder(); checkCancelledProcess(); - SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(theRequest.getLocation()); + SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(parentFolder); ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken()); if(theRequest instanceof TransferToThreddsRequest) { TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest; - WorkspaceItem item=request.getToTransfer(); - + Item item=request.getToTransfer(); + FileContainer itemContainer=client.open(item.getId()).asFile(); //look for metadata in same folder String itemName=item.getName(); reportItemName=itemName; String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml"; - WorkspaceItem metadataItem=getFileByName(item.getParent(),false,toLookMetadataName); + FileContainer metadataItem=getFileByName(client.open(item.getParentId()).asFolder(),false,toLookMetadataName); // if not present, generate with sis/geotk Destination toSetDestination=new Destination(); @@ -86,7 +94,7 @@ public class SynchronizationThread implements Runnable { toSetDestination.setPersistenceId(synchConfig.getRemotePersistence()); //NB ITEM IS SUPPOSED TO HAVE REMOTE PATH - String fileLocation=request.getLocation().getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + String fileLocation=request.getLocation().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+""; toSetDestination.setSubFolder(fileLocation); @@ -108,17 +116,17 @@ public class SynchronizationThread implements Runnable { checkCancelledProcess(); - TransferResult result=controller.transferFile(toSetDestination, item.getPublicLink(false), invocations); + TransferResult result=controller.transferFile(toSetDestination, itemContainer.getPublicLink(), invocations); - Map toSetProperties=new HashMap(); + Map toSetProperties=new HashMap(); String toSetMetadataUUID=null; Boolean validateMetadata=synchConfig.getValidateMetadata(); - - + + checkCancelledProcess(); if(metadataItem==null) { ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID); @@ -134,15 +142,18 @@ public class SynchronizationThread implements Runnable { toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID); toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+""); - item.getProperties().addProperties(toSetProperties); + org.gcube.common.storagehub.model.Metadata meta=item.getMetadata(); + meta.setMap(toSetProperties); + item.setMetadata(meta); + reportMessage="Successfully transferred and published."; // End ws->th } else if(theRequest instanceof TransferFromThreddsRequest) { - Map toSetProperties=new HashMap(); + Map toSetProperties=new HashMap(); TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest; String toImportName=null; - WorkspaceItem targetItem=null; + FileContainer targetItem=null; if(importRequest.getTargetItem()==null) { //Target Item will be created toImportName=importRequest.getRemoteFilename(); @@ -150,29 +161,38 @@ public class SynchronizationThread implements Runnable { }else { //Target Item already exists toImportName=importRequest.getTargetItem().getName(); - targetItem=importRequest.getTargetItem(); + targetItem=client.open(importRequest.getTargetItem().getId()).asFile(); reportMessage="Updating file.."; } reportItemName=toImportName; - + RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName); toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+""); InputStream source=null; try { source=controller.getInputStream(toImportName); - if(targetItem==null) - targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source); - else - targetItem.updateItem(source); - targetItem.getProperties().addProperties(toSetProperties); + targetItem=parentFolder.uploadFile(source, toImportName, "Imported from Thredds"); + // if(targetItem==null) + // targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source); + // else { + // targetItem.updateItem(source); + // } + + + org.gcube.common.storagehub.model.Metadata meta=targetItem.get().getMetadata(); + meta.setMap(toSetProperties); + targetItem.setMetadata(meta); }finally { if(source!=null) source.close(); } reportMessage="File successfully imported"; - + + + + }else if(theRequest instanceof DeleteRemoteRequest) { DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest; reportItemName=deleteRequest.getToRemoveName(); @@ -201,7 +221,7 @@ public class SynchronizationThread implements Runnable { log.debug("Remote File not found ",e); reportMessage="Remote File not found : "+e.getMessage(); toSetStatus=Status.ERROR; - }catch(InternalErrorException e) { + }catch(StorageHubException e) { log.debug("Internal generic exception ",e); reportMessage="Internal error : "+e.getMessage(); toSetStatus=Status.ERROR; @@ -210,7 +230,7 @@ public class SynchronizationThread implements Runnable { reportMessage="Unexpected exception : "+t.getMessage(); toSetStatus=Status.ERROR; }finally { - updateParentProperty(parentFolder, toSetStatus); + updateParentProperty(parentFolderItem, toSetStatus); submitReport(reportItemName,reportMessage,toSetStatus); ProcessIdProvider.instance.reset(); } @@ -218,19 +238,32 @@ public class SynchronizationThread implements Runnable { @Synchronized - private static void updateParentProperty(WorkspaceFolder folder,StepReport.Status toSetStatus) { - try { - String currentValue=folder.getProperties().getProperties().get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS); - if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null")) - folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"")); - else { - StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue); - if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus)) - folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"")); + private static void updateParentProperty(FolderItem folderItem,StepReport.Status toSetStatus) { + boolean retry=false; + do { + try { + retry=false; + FolderContainer folder=WorkspaceUtils.getClient().open(folderItem.getId()).asFolder(); + org.gcube.common.storagehub.model.Metadata folderMeta=folder.get().getMetadata(); + Map props=folderMeta.getMap(); + String currentValue=props.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+""; + if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null")) + props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""); + else { + StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue); + if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus)) + props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""); + } + + folderMeta.setMap(props); + folder.setMetadata(folderMeta); + }catch(ItemLockedException e) { + log.debug("Item locked retrying.. ",e); + retry=true; + }catch(Throwable t) { + log.warn("Unable to update folder status ",t); } - }catch(Throwable t) { - log.warn("Unable to update folder status ",t); - } + }while(retry); } @@ -254,11 +287,11 @@ public class SynchronizationThread implements Runnable { - private static MetadataReport publishMetadata(WorkspaceItem toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{ + private static MetadataReport publishMetadata(FileContainer toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{ File tempMetaFile=null; try { Metadata meta=SDIAbstractPlugin.metadata().build(); - tempMetaFile=NetUtils.download(toPublish.getPublicLink(false)); + tempMetaFile=NetUtils.download(toPublish.getPublicLink()); log.debug("Publishing metadata {} ",filename); @@ -274,14 +307,14 @@ public class SynchronizationThread implements Runnable { } - private static final WorkspaceItem getFileByName(WorkspaceFolder toLookIntoFolder,boolean recursive,String toLookForName) throws InternalErrorException { - log.debug("Looking for {} into {} [recursive {} ]",toLookForName,toLookIntoFolder.getPath()+" ID "+toLookIntoFolder.getId(),recursive); - for(WorkspaceItem item : toLookIntoFolder.getChildren()) - if(!item.isFolder()&&item.getName().equals(toLookForName)) return item; + private static final FileContainer getFileByName(FolderContainer toLookIntoFolder,boolean recursive,String toLookForName) throws StorageHubException { + log.debug(String.format("Looking for %1$s into %2$s [recursive %3$s ]",toLookForName,toLookIntoFolder.get().getPath()+" ID "+toLookIntoFolder.getId(),recursive)); + for(ItemContainer item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers()) + if(!item.getType().equals(ContainerType.FOLDER)&&item.get().getName().equals(toLookForName)) return (FileContainer) item; if(recursive) { - for(WorkspaceItem item : toLookIntoFolder.getChildren()) - if(item.isFolder()) return getFileByName((WorkspaceFolder) item, recursive, toLookForName); + for(ItemContainer item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers()) + if(item.getType().equals(ContainerType.FOLDER)) return getFileByName((FolderContainer) item, recursive, toLookForName); } return null; diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java index 160cfd7..57093c6 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java @@ -1,23 +1,32 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; +import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.engine.impl.Process; import lombok.Data; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j public class TransferFromThreddsRequest extends SynchronizationRequest { - private WorkspaceItem targetItem; // can be null + private Item targetItem; // can be null private String remoteFilename; - public TransferFromThreddsRequest(Process process, WorkspaceItem targetItem, WorkspaceFolder containingFolder, + public TransferFromThreddsRequest(Process process, Item targetItem, FolderItem containingFolder, String remoteFilename) { super(process,containingFolder); this.targetItem = targetItem; this.remoteFilename = remoteFilename; + if(targetItem==null) + log.debug(String.format("Created IMPORT request of %1$s into %1$s from %3$s", + remoteFilename,containingFolder.getPath(),containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH))); + else + log.debug(String.format("Created UPDATE %1$s from %2$s", targetItem.getPath(), + containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH))); } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java index 42bbcaf..ef8d17b 100644 --- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java @@ -1,23 +1,27 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; +import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.engine.impl.Process; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; @Getter @Setter - +@Slf4j public class TransferToThreddsRequest extends SynchronizationRequest { - private WorkspaceItem toTransfer; + private Item toTransfer; - public TransferToThreddsRequest(Process process,WorkspaceFolder location, WorkspaceItem toTransfer) { + public TransferToThreddsRequest(Process process,FolderItem location, Item toTransfer) { super(process,location); this.toTransfer = toTransfer; + log.debug(String.format("Created SEND request from $1%s to $2%s", + toTransfer.getPath(),location.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH))); } } diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java new file mode 100644 index 0000000..25efa86 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class ItemNotFoundException extends WorkspaceInteractionException { + + /** + * + */ + private static final long serialVersionUID = -6818076224344812668L; + + public ItemNotFoundException() { + // TODO Auto-generated constructor stub + } + + public ItemNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public ItemNotFoundException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public ItemNotFoundException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public ItemNotFoundException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java new file mode 100644 index 0000000..ce1c04e --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java @@ -0,0 +1,18 @@ +package org.gcube.usecases.ws.thredds.model; + +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; + +@Getter +@AllArgsConstructor +@ToString +public class SyncEngineStatusDescriptor { + + private int parallelTransferCount; + private int queueLenght; + private Map configuration; + +} diff --git a/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties b/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties index ccdd3c5..30064f1 100644 --- a/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties +++ b/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties @@ -1,6 +1,6 @@ scanner.pool.maxSize=10 -scanner.pool.coreSize=1 +scanner.pool.coreSize=10 scanner.pool.idle.ms=30000 transfers.pool.maxSize=40 -transfers.pool.coreSize=1 +transfers.pool.coreSize=20 transfers.pool.idle.ms=30000 \ No newline at end of file diff --git a/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java b/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java index 5121a67..a894a24 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java @@ -1,40 +1,12 @@ package org.gcube.usecases.ws.thredds; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Collections; - -import org.apache.commons.io.IOUtils; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; -import org.gcube.data.transfer.library.DataTransferClient; -import org.gcube.data.transfer.library.faults.DestinationNotSetException; -import org.gcube.data.transfer.library.faults.FailedTransferException; -import org.gcube.data.transfer.library.faults.InitializationException; -import org.gcube.data.transfer.library.faults.InvalidDestinationException; -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.faults.ServiceNotFoundException; -import org.gcube.data.transfer.library.faults.SourceNotSetException; -import org.gcube.data.transfer.library.faults.UnreachableNodeException; -import org.gcube.data.transfer.model.Destination; -import org.gcube.data.transfer.model.DestinationClashPolicy; -import org.gcube.data.transfer.model.PluginInvocation; -import org.gcube.usecases.ws.thredds.faults.InternalException; -import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; public class DTTests { - public static void main(String[] args) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException, MalformedURLException, FileNotFoundException, IOException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { + public static void main(String[] args) throws Exception { // TestCommons.setScope(); // String threddsHostname="thredds-d-d4s.d4science.org"; // DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+threddsHostname+":80"); @@ -57,8 +29,10 @@ public class DTTests { // TokenSetter.set("/gcube/devNext"); - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - String folderId=ws.getItemByPath("/Workspace/Accounting").getId(); +// Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + StorageHubClient client=WorkspaceUtils.getClient(); + + String folderId=TestCommons.getByPath("/Workspace/Accounting").getId(); SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId); System.out.println("Done"); diff --git a/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java b/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java index 469c9db..c93885d 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java @@ -1,44 +1,35 @@ package org.gcube.usecases.ws.thredds; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; -import org.gcube.common.homelibrary.home.workspace.folder.FolderItem; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FileContainer; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; public class GetWSSynchSpace { - public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { - TokenSetter.set("/gcube/preprod/preVRE"); + public static void main(String[] args) throws Exception { + TokenSetter.set("/gcube"); SyncEngine engine=SyncEngine.get(); - String folderId=TestCommons.getWSIdByPath("/Workspace/Thredds main catalog"); + FolderContainer folder=(FolderContainer) TestCommons.getByPath("ThreddsDev"); - System.out.println(folderId); - - - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - WorkspaceFolder folder=(WorkspaceFolder) ws.getItem(folderId); + System.out.println(folder.getId()); SynchFolderConfiguration config=new SynchFolderConfiguration(); System.out.println("Total size : "+computeLength(folder, config)); } - private static final long computeLength(WorkspaceItem item,SynchFolderConfiguration config) throws InternalErrorException { + private static final long computeLength(ItemContainer item,SynchFolderConfiguration config) throws StorageHubException{ long toReturn=0l; - if(item.isFolder()) { - for(WorkspaceItem child:((WorkspaceFolder)item).getChildren()) { - if(item.isFolder()||config.matchesFilter(item.getName())) + if(item.getType().equals(ContainerType.FOLDER)) { + for(ItemContainer child:((FolderContainer)item).list().withContent().getContainers()) { + if(child.getType().equals(ContainerType.FOLDER)||config.matchesFilter(child.get().getName())) toReturn=toReturn+computeLength(child, config); } - }else toReturn=toReturn+((FolderItem)item).getLength(); + }else toReturn=toReturn+((FileContainer)item).get().getContent().getSize(); return toReturn; } diff --git a/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java b/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java index a3e57ee..cf98313 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java @@ -4,23 +4,14 @@ import java.io.File; import java.util.Map; import java.util.Map.Entry; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Properties; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FileContainer; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; +import org.gcube.common.storagehub.model.items.Item; import org.gcube.data.transfer.library.DataTransferClient; -import org.gcube.data.transfer.library.faults.DestinationNotSetException; -import org.gcube.data.transfer.library.faults.FailedTransferException; -import org.gcube.data.transfer.library.faults.InitializationException; -import org.gcube.data.transfer.library.faults.InvalidDestinationException; -import org.gcube.data.transfer.library.faults.InvalidSourceException; -import org.gcube.data.transfer.library.faults.SourceNotSetException; import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.spatial.data.sdi.interfaces.Metadata; @@ -28,16 +19,17 @@ import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions; import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder; import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; public class PublicLinkIssueTest { - public static void main (String[] args) throws ItemNotFoundException, InternalErrorException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { + public static void main (String[] args) throws Exception { TokenSetter.set("/d4science.research-infrastructures.eu"); - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - System.out.println(ws.getItemByPath("/Workspace/ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc").getPublicLink(false)); + StorageHubClient shClient=WorkspaceUtils.getClient(); + System.out.println(((FileContainer)TestCommons.getByPath("ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc")).getPublicLink()); String threddsHostName = Commons.getThreddsHost(); @@ -51,7 +43,7 @@ public class PublicLinkIssueTest { client.httpSource("http://data.d4science.org/V2drR2gxSFRTQlpLVC9nakozL29QcDdPR2U5UEVHYWRHbWJQNStIS0N6Yz0", dest); - scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d")); + scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder()); System.out.println("OK FIRST ..."); try { Metadata meta=SDIAbstractPlugin.metadata().build(); @@ -65,29 +57,29 @@ public class PublicLinkIssueTest { e.printStackTrace(); } System.out.println("CHECKING AGAIN"); - scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d")); + scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder()); } - public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException { - System.out.println("Folder "+folder.getPath()); - printProperties(folder.getProperties()); + public static void scanForPrint(FolderContainer folder) throws StorageHubException { + System.out.println("Folder "+folder.get().getPath()); + printProperties(folder.get()); SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "",""); - for(WorkspaceItem item:folder.getChildren()) - if(!item.isFolder()&&config.matchesFilter(item.getName())) { + for(ItemContainer item:folder.list().withMetadata().getContainers()) + if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) { // System.out.println("ITEM "+item.getPath()); - printProperties(item.getProperties()); + printProperties(item.get()); } - for(WorkspaceItem item:folder.getChildren()) - if(item.isFolder())scanForPrint((WorkspaceFolder) item); + for(ItemContainer item:folder.list().withMetadata().getContainers()) + if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item); } - public static void printProperties(Properties prop) throws InternalErrorException { - Map map=prop.getProperties(); + public static void printProperties(Item toPrint){ + Map map=toPrint.getMetadata().getMap(); // System.out.print("Properties : .."); - for(Entry entry:map.entrySet()) { + for(Entry entry:map.entrySet()) { if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)|| entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)|| entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) { diff --git a/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java b/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java index 871383e..3200c5f 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java @@ -1,21 +1,13 @@ package org.gcube.usecases.ws.thredds; -import java.io.IOException; -import java.net.MalformedURLException; import java.util.HashMap; import java.util.Map; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; @@ -35,9 +27,9 @@ public class TestCommons { String toCreateCatalogName; - public String getFolderId() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + public String getFolderId() throws Exception { if(folderId.startsWith("/")) - return getWSIdByPath(folderId); + return getByPath(folderId).getId(); else return folderId; } @@ -45,26 +37,46 @@ public class TestCommons { private static Map configs=new HashMap<>(); - private static String toUseConfig="pre"; + private static String toUseConfig="default"; static { - configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","","")); +// configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","","")); // folderName="WS-Tests"; - configs.put("default", new TestSet("Default Tests","/gcube/devsec/devVRE","/Workspace/ThreddsDev","public/netcdf","***REMOVED***","main")); + configs.put("simple", new TestSet("Simple label ","/gcube", "Test1","public/netcdf/simpleFolder","***REMOVED***","simple")); + configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems")); + configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main")); - configs.put("pre", new TestSet("Default Tests","/gcube/preprod/preVRE","/Workspace/CMEMS","public/netcdf/CMEMS","***REMOVED***","main")); +// configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main")); +// +// +// configs.put("pre", new TestSet("Default Tests","/pred4s/preprod/preVRE","Thredds Synch","Fabio WS","91a8433b-9491-42c1-95e2-420245ef1943-980114272","SynchTestCatalog")); } - public static String getWSIdByPath(String path) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - return ws.getItemByPath(path).getId(); + public static ItemContainer getByPath(String path) throws Exception { + StorageHubClient client = WorkspaceUtils.getClient(); + FolderContainer root=client.getWSRoot(); + + return scan(root,path); + } + + + public static ItemContainer scan(FolderContainer folder,String path) throws Exception{ + String toLookFor=path.substring((path.startsWith("/")?1:0), path.length()); + String[] split=toLookFor.split("/"); + toLookFor=split[0]; + for(ItemContainer item:folder.list().withMetadata().getContainers()) + if(item.get().getName().equals(toLookFor)) { + if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length())); + else return item; + } + throw new Exception("Unable to find "+path); } @@ -73,13 +85,13 @@ public class TestCommons { } - public static WorkspaceFolder getTestFolder() throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, IOException { - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + public static FolderContainer getTestFolder() throws Exception { + try{ - return (WorkspaceFolder) ws.getItem(configs.get(toUseConfig).getFolderId()); + return WorkspaceUtils.getClient().open(configs.get(toUseConfig).getFolderId()).asFolder(); }catch(Throwable t) { // try to use path - return (WorkspaceFolder) ws.getItemByPath(configs.get(toUseConfig).getFolderId()); + return (FolderContainer) getByPath(configs.get(toUseConfig).getFolderId()); } @@ -101,12 +113,12 @@ public class TestCommons { } - public static ThreddsController getThreddsController() throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + public static ThreddsController getThreddsController() throws Exception { SynchFolderConfiguration config=getSynchConfig(); return new ThreddsController(config.getRemotePath(), config.getTargetToken()); } - public static SynchFolderConfiguration getSynchConfig() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + public static SynchFolderConfiguration getSynchConfig() throws Exception { TestSet set=configs.get(toUseConfig); return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId()); } diff --git a/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java index 9aed7fd..0d3fb5d 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java @@ -1,18 +1,12 @@ package org.gcube.usecases.ws.thredds; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; -import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; public class ThreddsTests { - public static void main(String[] args) throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + public static void main(String[] args) throws Exception{ TestCommons.setScope(); SynchFolderConfiguration folderConfig=TestCommons.getSynchConfig(); diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java b/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java index 8b1b0ed..f0b2971 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java @@ -1,40 +1,31 @@ package org.gcube.usecases.ws.thredds; -import java.io.IOException; -import java.net.MalformedURLException; - -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; public class WSTimes { - public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, InternalErrorException, HomeNotFoundException, UserNotFoundException, IOException { + public static void main(String[] args) throws Exception { TestCommons.setScope(); - WorkspaceFolder folder=TestCommons.getTestFolder(); - for(WorkspaceItem item : folder.getChildren()) { + FolderContainer folder=TestCommons.getTestFolder(); + for(ItemContainer item : folder.list().withMetadata().getContainers()) { printDates(item); try { Thread.sleep(3000); } catch (InterruptedException e) { } - item.getProperties().addProperty("Fake prop", "fake value"); + WorkspaceUtils.addParameter(item,"Fake prop", "fake value"); printDates(item); } } - public static final void printDates(WorkspaceItem item) throws InternalErrorException { - System.out.println("ITEM : "+item.getName()); - System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getCreationTime().getTime())); - System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getLastModificationTime().getTime())); + public static final void printDates(ItemContainer item) { + System.out.println("ITEM : "+item.get().getName()); + System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getCreationTime().getTime())); + System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getLastModificationTime().getTime())); } } diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java index 77f96fc..e513098 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java @@ -1,24 +1,14 @@ package org.gcube.usecases.ws.thredds; -import java.io.IOException; import java.util.Date; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; public class WorkspaceAccounting { - public static void main(String[] args) throws InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, IOException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, NumberFormatException, ItemNotFoundException { + public static void main(String[] args) throws Exception { TestCommons.setScope(); - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + // WorkspaceFolder folder=ws.getRoot().createFolder("Accounting", "test purposes"); @@ -68,7 +58,7 @@ public class WorkspaceAccounting { - System.out.println(WorkspaceUtils.isModifiedAfter(ws.getItemByPath(TestCommons.getTestFolder().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc"), new Date(0l))); + System.out.println(WorkspaceUtils.isModifiedAfter(TestCommons.getByPath(TestCommons.getTestFolder().get().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc").get(), new Date(0l))); } diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java index b655557..599b6d1 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java @@ -1,28 +1,24 @@ package org.gcube.usecases.ws.thredds; -import java.io.IOException; -import java.net.MalformedURLException; - -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; -import org.gcube.usecases.ws.thredds.faults.InternalException; -import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.model.Metadata; public class WorkspaceCleanup { - public static void main(String[] args) throws WorkspaceInteractionException, InternalException, InternalErrorException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, HomeNotFoundException, UserNotFoundException, IOException { + public static void main(String[] args) throws Exception{ TestCommons.setScope(); - WorkspaceFolder folder=TestCommons.getTestFolder(); + FolderContainer folder=TestCommons.getTestFolder(); + + Metadata meta=folder.get().getMetadata(); + meta.setMap(Constants.cleanedFolderPropertiesMap); + folder.setMetadata(meta); SyncEngine engine=SyncEngine.get(); engine.forceUnlock(folder.getId()); - engine.unsetSynchronizedFolder(folder.getId(), false); +// engine.unsetSynchronizedFolder(folder.getId(), false); + +// + WorkspaceProperties.printProperties(folder); } } diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java index 2a92729..27e9002 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java @@ -1,36 +1,23 @@ package org.gcube.usecases.ws.thredds; -import java.io.IOException; -import java.net.MalformedURLException; import java.util.Collections; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager; -import org.gcube.usecases.ws.thredds.faults.InternalException; -import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; -import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; -import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; public class WorkspaceLock { - public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, HomeNotFoundException, UserNotFoundException, WorkspaceInteractionException, InternalException, MalformedURLException, IOException { + public static void main(String[] args) throws Exception { TestCommons.setScope(); - WorkspaceFolder folder=TestCommons.getTestFolder(); + FolderContainer folder=TestCommons.getTestFolder(); WorkspaceFolderManager manager=new WorkspaceFolderManager(folder.getId()); String processID="mytest"; try { System.out.println("Trying to cleanup, first.. "); - folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null)); - System.out.println("FOLDER PROPERTIES : "+folder.getProperties().getProperties()); + WorkspaceUtils.addParameters(folder, Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null)); + System.out.println("FOLDER PROPERTIES : "+folder.get().getMetadata().getMap()); }catch(Throwable t) { System.err.println("Dismiss error : "); t.printStackTrace(); diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java index 737e094..7daf454 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java @@ -1,35 +1,29 @@ package org.gcube.usecases.ws.thredds; -import java.io.IOException; -import java.net.MalformedURLException; import java.util.Map; import java.util.Map.Entry; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Properties; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; -import org.gcube.usecases.ws.thredds.faults.InternalException; -import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; public class WorkspaceProperties { - public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, ItemNotFoundException, MalformedURLException, IOException, WorkspaceInteractionException, InternalException { + public static void main(String[] args) throws Exception{ TestCommons.setScope(); - WorkspaceFolder folder=TestCommons.getTestFolder(); - + FolderContainer folder=TestCommons.getTestFolder(); +// System.out.println("Props : "+folder.get().getMetadata().getMap().size()); +// printProperties(folder); +// System.out.println(new WorkspaceFolderManager(folder.getId()).isSynched()); SyncEngine.get().check(folder.getId(), true); scanForPrint(folder); + // SyncEngine.get().shutDown(); // for(Workspace) // printProperties(folder.getProperties()); @@ -61,24 +55,24 @@ public class WorkspaceProperties { } - public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException { - System.out.println("Folder "+folder.getPath()); - printProperties(folder.getProperties()); + public static void scanForPrint(FolderContainer folder) throws StorageHubException { + System.out.println("Folder "+folder.get().getPath()); + printProperties(folder); SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "",""); - for(WorkspaceItem item:folder.getChildren()) - if(!item.isFolder()&&config.matchesFilter(item.getName())) { - System.out.println("ITEM "+item.getPath()); - printProperties(item.getProperties()); + for(ItemContainer item:folder.list().withMetadata().getContainers()) + if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) { + System.out.println("ITEM "+item.get().getPath()); + printProperties(item); } - for(WorkspaceItem item:folder.getChildren()) - if(item.isFolder())scanForPrint((WorkspaceFolder) item); + for(ItemContainer item:folder.list().withMetadata().getContainers()) + if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item); } - public static void printProperties(Properties prop) throws InternalErrorException { - Map map=prop.getProperties(); + public static void printProperties(ItemContainer item) { + Map map=item.get().getMetadata().getMap(); System.out.print("Properties : .."); - for(Entry entry:map.entrySet()) { + for(Entry entry:map.entrySet()) { // if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)|| // entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)|| // entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) { diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java index 2f23942..f9516ee 100644 --- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java @@ -1,23 +1,10 @@ package org.gcube.usecases.ws.thredds; -import java.io.IOException; -import java.net.MalformedURLException; import java.util.concurrent.Semaphore; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus; -import org.gcube.usecases.ws.thredds.faults.InternalException; -import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException; -import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException; import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; @@ -25,18 +12,20 @@ import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; public class WorkspaceSynchronizationTest { - public static void main(String[] args) throws ProcessNotFoundException, InternalErrorException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, HomeNotFoundException, UserNotFoundException, MalformedURLException, IOException { + public static void main(String[] args) throws Exception { // GET ENGINE : SINGLETON INSTANCE - SyncEngine engine=SyncEngine.get(); + final SyncEngine engine=SyncEngine.get(); engine.setRequestLogger("requests.txt"); //TEST INFO... TestCommons.setScope(); - WorkspaceFolder folder=TestCommons.getTestFolder(); + FolderContainer folder=TestCommons.getTestFolder(); // FOLDER CONFIGURATION BEAN SynchFolderConfiguration config=TestCommons.getSynchConfig(); +// SynchFolderConfiguration config=null; + // try { // //try to clean it up, first.. @@ -49,38 +38,40 @@ public class WorkspaceSynchronizationTest { // engine.unsetSynchronizedFolder(folder.getId(), false); // } - + String folderId=folder.getId(); +// String folderId="15db7b0e-c215-41b7-b384-a1deda250b65"; try { // WHEN OPENING A FOLDER, INVOKE CHECK TO UPDATE SYNCH STATUS - engine.check(folder.getId(), false); + engine.check(folderId, false); }catch(WorkspaceNotSynchedException e) { System.out.println("Folder not synched, configurin it.."); - engine.setSynchronizedFolder(config, folder.getId()); - engine.check(folder.getId(), false); + engine.setSynchronizedFolder(config, folderId); + engine.check(folderId, false); }catch(WorkspaceLockedException e) { System.out.println("Workspace locked, going to force unlock.."); // MAINLY FOR TEST PURPOSES, OR WHEN SOMETHIGN GOES WRONG.. USE CAUTIOUSLY - engine.forceUnlock(folder.getId()); - engine.check(folder.getId(), false); + engine.forceUnlock(folderId); + engine.check(folderId, false); } // INVOKE SYNCHRONIZATION ON FOLDER - ProcessDescriptor descriptor=engine.doSync(folder.getId()); + ProcessDescriptor descriptor=engine.doSync(folderId); System.out.println("Obtained descriptor : "+descriptor); Semaphore sem=new Semaphore(0); // REGISTER CALLBACK TO MONITOR PROGRESS - engine.registerCallBack(folder.getId(), new SyncOperationCallBack() { + engine.registerCallBack(folderId, new SyncOperationCallBack() { @Override public void onStep(ProcessStatus status, ProcessDescriptor descriptor) { System.out.println("ON STEP : "+status+" "+descriptor); + System.out.println("ENGINE STATUS : "+engine.getStatus()); System.out.println("LOG : \n"+ status.getLogBuilder().toString()); if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release(); } @@ -93,7 +84,7 @@ public class WorkspaceSynchronizationTest { } - engine.check(folder.getId(), true); + engine.check(folderId, true); // INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES