diff --git a/pom.xml b/pom.xml
index f728b39..cad98ea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,17 +4,17 @@
org.gcube.tools
maven-parent
- LATEST
+ 1.1.0
- org.gcube.spatial-data
+ org.gcube.spatial.data
ws-thredds
- 0.2.0-SNAPSHOT
+ 0.2.2-SNAPSHOT
ws-thredds
prototype of WS integration with data-transfer for Thredds pubblication
${project.basedir}/distro
- http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId}
+ http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId}
@@ -24,7 +24,17 @@
${svnBaseUrl}/${project.artifactId}
-
+
+
+
+ org.gcube.distribution
+ maven-portal-bom
+ 3.4.0-SNAPSHOT
+ pom
+ import
+
+
+
@@ -32,38 +42,38 @@
org.gcube.spatial.data
sdi-library
[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
-
+
org.gcube.data.transfer
data-transfer-library
[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)
-
-
-
-
-
-
- org.gcube.common
- home-library-jcr
- [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)
-
+
+
+
+
org.gcube.common
- home-library
- [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)
+ storagehub-client-library
-
+
org.slf4j
- slf4j-log4j12
- 1.6.4
- provided
+ slf4j-log4j12
-
-
+
+
\ No newline at end of file
diff --git a/requests.txt b/requests.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/Commons.java b/src/main/java/org/gcube/usecases/ws/thredds/Commons.java
index 28dcbb9..c2d0601 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/Commons.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/Commons.java
@@ -14,7 +14,7 @@ import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
-import org.apache.tika.io.IOUtils;
+import org.apache.commons.io.IOUtils;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.client.AuthorizationFilter;
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/Constants.java b/src/main/java/org/gcube/usecases/ws/thredds/Constants.java
index 3b81177..6e5df5e 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/Constants.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/Constants.java
@@ -55,11 +55,11 @@ public class Constants {
public static final String TRANSFERS_POOL_IDLE_MS="transfers.pool.idle.ms";
}
- public static final Map cleanedItemPropertiesMap=new HashMap();
+ public static final Map cleanedItemPropertiesMap=new HashMap();
- public static final Map cleanedFolderPropertiesMap=new HashMap();
+ public static final Map cleanedFolderPropertiesMap=new HashMap();
- public static final Map defaultConfigurationMap=new HashMap();
+ public static final Map defaultConfigurationMap=new HashMap();
static {
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.TBS, null);
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java b/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java
index 10a2378..34107af 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java
@@ -1,5 +1,7 @@
package org.gcube.usecases.ws.thredds;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import lombok.Synchronized;
@@ -10,7 +12,7 @@ public class LocalConfiguration {
private static LocalConfiguration instance=null;
@Synchronized
- private static final LocalConfiguration get() {
+ public static final LocalConfiguration get() {
if(instance==null)
instance=new LocalConfiguration();
return instance;
@@ -21,16 +23,22 @@ public class LocalConfiguration {
return (String) get().props.getOrDefault(property, Constants.defaultConfigurationMap.get(property));
}catch(Throwable t) {
log.warn("Unable to get configuration property "+property,t);
- return Constants.defaultConfigurationMap.get(property);
+ return Constants.defaultConfigurationMap.get(property)+"";
}
}
+ public Map asMap(){
+ HashMap toReturn=new HashMap<>();
+ for(Object key :props.keySet())
+ toReturn.put(key+"", (String)props.getOrDefault(key, Constants.defaultConfigurationMap.get(key)));
+ return toReturn;
+ }
//***************** INSTANCE
Properties props;
- public LocalConfiguration() {
+ private LocalConfiguration() {
props=new Properties();
try{
props.load(this.getClass().getResourceAsStream("configuration.properties"));
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java b/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java
index 9627bec..c347923 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java
@@ -38,20 +38,19 @@ public class NetUtils {
}
- public static String resolveRedirects(String url) throws IOException{
+ public static URL resolveRedirects(URL url) throws IOException{
log.debug("Resolving redirect for url {} ",url);
- URL urlObj=new URL(url);
- HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
int status=connection.getResponseCode();
if(status>=300&&status<400){
String newUrl=connection.getHeaderField("Location");
log.debug("Following redirect from {} to {} ",url,newUrl);
- return resolveRedirects(newUrl);
+ return resolveRedirects(new URL(newUrl));
}else return url;
}
- public static File download(String url) throws MalformedURLException, IOException {
- return toFile(new URL(resolveRedirects(url)).openStream());
+ public static File download(URL url) throws MalformedURLException, IOException {
+ return toFile(resolveRedirects(url).openStream());
}
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java b/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java
index 035d661..ce175e2 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java
@@ -1,283 +1,258 @@
package org.gcube.usecases.ws.thredds;
-import java.io.File;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.gcube.common.homelibrary.home.HomeLibrary;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Workspace;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WrongItemTypeException;
-import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
-import org.gcube.usecases.ws.thredds.engine.PublishRequest;
-import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
-import org.gcube.usecases.ws.thredds.engine.PublishRequest.PublishItem;
-import org.gcube.usecases.ws.thredds.engine.TransferRequestServer;
-import org.gcube.usecases.ws.thredds.engine.TransferRequestServer.Report;
-
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class PublishFolders {
- private static String WS_USER="fabio.sinibaldi";
-
-
- public static void main(String[] args) {
- TokenSetter.set("/d4science.research-infrastructures.eu");
- HashSet configs=new HashSet<>();
-
-
-// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS
- // String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files
- String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST
-
-// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET
-
- String publishingUserToken="***REMOVED***"; //fabio @NextNext
-
-
- FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE");
- // folderConfig.setProvidedMetadata(true);
- // folderConfig.setMetadataFolderId(metadataFolderID);
-
-
-
-
- configs.add(folderConfig);
-
-
-
-
-
-
-
- TransferRequestServer server=new TransferRequestServer();
- for(FolderConfiguration entry:configs){
- try{
- Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace();
- // FolderReport report=new FolderReport(entry);
- log.info("Managing {} ",entry);
- WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId());
- handleFolder(ws,entry,server,folder);
-
-
-
- }catch(WorkspaceException e){
- System.err.println("WORKSPACE EXC ");
- e.printStackTrace(System.err);
- }catch(HomeNotFoundException e){
- System.err.println("WORKSPACE EXC ");
- e.printStackTrace(System.err);
- }catch(InternalErrorException e){
- System.err.println("WORKSPACE EXC ");
- e.printStackTrace(System.err);
- }catch(UserNotFoundException e){
- System.err.println("WORKSPACE EXC ");
- e.printStackTrace(System.err);
- }catch(Exception e){
- System.err.println("UNEXPECTED EXC");
- e.printStackTrace(System.err);
- }
- }
- System.out.println("Waiting for service.. ");
- server.waitCompletion();
-
- Report report=server.getReport();
-
- File reportFile =report.toFile(folderConfig);
- System.out.println("Report at "+reportFile.getAbsolutePath());
-
-
- }
-
- /**
- * For *.nc | *.ncml
- * if relatedMetadataFolder contains .xml use meta
- * else ask DT to generate it
- *
- *
- * @param wsFolder
- * @param config
- * @param server
- * @return
- * @throws WorkspaceException
- * @throws InternalErrorException
- * @throws ItemNotFoundException
- * @throws HomeNotFoundException
- * @throws UserNotFoundException
- */
-
- public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
-
- log.info("Handling folder {} ",folder.getPath());
-
- cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName());
-
- log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath());
- publishFolder(ws,config,server,folder);
- }
-
-
- public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
-
-
-
-
- //Access folder
- List folderItems=folder.getChildren();
-
- if(config.isIncludeSubfolders()) {
- log.info("Going through subfolders first.....");
- for(WorkspaceItem item:folderItems) {
- try {
- if(item.isFolder()) {
- FolderConfiguration subConfig=new FolderConfiguration(config);
- subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName());
- publishFolder(ws,subConfig,server,(WorkspaceFolder) item);
- log.debug("Subfolder {} successfully analyzed ",item.getPath());
- }
- }catch(Exception e) {
- try{
- log.warn("Unabel to check item {} ",item.getPath(),e);
- }catch(InternalErrorException e1) {
- log.warn("Unabel to check item and to get Path {} ",item,e);
- }
- }
- }
- }
-
-
-
- log.debug("Checking for ncml files .... ");
- for(WorkspaceItem item : folderItems) {
- try {
- if(!item.isFolder()) {
- String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
- if(prefix.equals(".ncml")) {
- PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken());
- if(config.isProvidedMetadata()) {
- String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
- File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
- if (meta!=null) req.setMetadata(meta);
- }
- // TODO NB Check for queue
- server.put(req);
- }
- }
- }catch(Exception e) {
- try{
- log.warn("Unabel to check item {} ",item.getPath(),e);
- }catch(InternalErrorException e1) {
- log.warn("Unabel to check item and to get Path {} ",item,e);
- }
- }
- }
-
- log.debug("Checking nc files.. ");
-
- for(WorkspaceItem item:folder.getChildren()){
- try {
- if(!item.isFolder()) {
- String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
- if(prefix.equals(".nc")){
- // NC
- PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken());
- if(config.isProvidedMetadata()) {
- String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
- File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
- if (meta!=null) req.setMetadata(meta);
- }
- server.put(req);
- }
- }
- }catch(Exception e) {
- try{
- log.warn("Unabel to check item {} ",item.getPath(),e);
- }catch(InternalErrorException e1) {
- log.warn("Unabel to check item and to get Path {} ",item,e);
- }
- }
- }
- folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+""));
- log.debug("Creating requests... ");
- }
-
- private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException {
- try{
- WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId);
- if(found==null) throw new ItemNotFoundException("Found item was null");
- return NetUtils.toFile(((ExternalFile)found).getData());
- }catch(ItemNotFoundException e) {
- return null;
- }
- }
-
- // Remotely deletes folders which has history operations : RENAMING, REMOVAL
- private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException {
- List history=folder.getAccounting();
- long lastUpdateTimeMillis=getLastUpdateTime(folder);
- Date lastUpdate=new Date(lastUpdateTimeMillis);
- log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate));
-
-
- //look into history
- boolean toDeleteCurrentFolder=false;
- for(AccountingEntry entry: history) {
- Date eventTime=entry.getDate().getTime();
-
- switch(entry.getEntryType()) {
- case REMOVAL:
-
- case RENAMING:{
- log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
- if(eventTime.after(lastUpdate)) {
- log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
- toDeleteCurrentFolder=true;
- }
- }
- }
- if(toDeleteCurrentFolder) break;
- }
-
-
- //Delete Folder or scan children
- if(toDeleteCurrentFolder) {
- log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath);
- try{
- Commons.cleanupFolder(remoteFolderPath,targetToken);
- }catch(Throwable t) {
- log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t);
- if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish
- throw t;
-
- }
- }else {
- log.info("Folder is not to be cleaned up. Checking children..");
- for(WorkspaceItem item:folder.getChildren())
- if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName());
- }
- }
-
- private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException {
- try{
- return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME));
- }catch(Throwable e) {
- log.debug("Unable to get last update time for {}. Considering 0..",item.getPath());
- return 0l;
- }
- }
-
+// private static String WS_USER="fabio.sinibaldi";
+//
+//
+// public static void main(String[] args) {
+// TokenSetter.set("/d4science.research-infrastructures.eu");
+// HashSet configs=new HashSet<>();
+//
+//
+//// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS
+// // String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files
+// String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST
+//
+//// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET
+//
+// String publishingUserToken="***REMOVED***"; //fabio @NextNext
+//
+//
+// FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE");
+// // folderConfig.setProvidedMetadata(true);
+// // folderConfig.setMetadataFolderId(metadataFolderID);
+//
+//
+//
+//
+// configs.add(folderConfig);
+//
+//
+//
+//
+//
+//
+//
+// TransferRequestServer server=new TransferRequestServer();
+// for(FolderConfiguration entry:configs){
+// try{
+// Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace();
+// // FolderReport report=new FolderReport(entry);
+// log.info("Managing {} ",entry);
+// WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId());
+// handleFolder(ws,entry,server,folder);
+//
+//
+//
+// }catch(WorkspaceException e){
+// System.err.println("WORKSPACE EXC ");
+// e.printStackTrace(System.err);
+// }catch(HomeNotFoundException e){
+// System.err.println("WORKSPACE EXC ");
+// e.printStackTrace(System.err);
+// }catch(InternalErrorException e){
+// System.err.println("WORKSPACE EXC ");
+// e.printStackTrace(System.err);
+// }catch(UserNotFoundException e){
+// System.err.println("WORKSPACE EXC ");
+// e.printStackTrace(System.err);
+// }catch(Exception e){
+// System.err.println("UNEXPECTED EXC");
+// e.printStackTrace(System.err);
+// }
+// }
+// System.out.println("Waiting for service.. ");
+// server.waitCompletion();
+//
+// Report report=server.getReport();
+//
+// File reportFile =report.toFile(folderConfig);
+// System.out.println("Report at "+reportFile.getAbsolutePath());
+//
+//
+// }
+//
+// /**
+// * For *.nc | *.ncml
+// * if relatedMetadataFolder contains .xml use meta
+// * else ask DT to generate it
+// *
+// *
+// * @param wsFolder
+// * @param config
+// * @param server
+// * @return
+// * @throws WorkspaceException
+// * @throws InternalErrorException
+// * @throws ItemNotFoundException
+// * @throws HomeNotFoundException
+// * @throws UserNotFoundException
+// */
+//
+// public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
+//
+// log.info("Handling folder {} ",folder.getPath());
+//
+// cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName());
+//
+// log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath());
+// publishFolder(ws,config,server,folder);
+// }
+//
+//
+// public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
+//
+//
+//
+//
+// //Access folder
+// List folderItems=folder.getChildren();
+//
+// if(config.isIncludeSubfolders()) {
+// log.info("Going through subfolders first.....");
+// for(WorkspaceItem item:folderItems) {
+// try {
+// if(item.isFolder()) {
+// FolderConfiguration subConfig=new FolderConfiguration(config);
+// subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName());
+// publishFolder(ws,subConfig,server,(WorkspaceFolder) item);
+// log.debug("Subfolder {} successfully analyzed ",item.getPath());
+// }
+// }catch(Exception e) {
+// try{
+// log.warn("Unabel to check item {} ",item.getPath(),e);
+// }catch(InternalErrorException e1) {
+// log.warn("Unabel to check item and to get Path {} ",item,e);
+// }
+// }
+// }
+// }
+//
+//
+//
+// log.debug("Checking for ncml files .... ");
+// for(WorkspaceItem item : folderItems) {
+// try {
+// if(!item.isFolder()) {
+// String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
+// if(prefix.equals(".ncml")) {
+// PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken());
+// if(config.isProvidedMetadata()) {
+// String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
+// File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
+// if (meta!=null) req.setMetadata(meta);
+// }
+// // TODO NB Check for queue
+// server.put(req);
+// }
+// }
+// }catch(Exception e) {
+// try{
+// log.warn("Unabel to check item {} ",item.getPath(),e);
+// }catch(InternalErrorException e1) {
+// log.warn("Unabel to check item and to get Path {} ",item,e);
+// }
+// }
+// }
+//
+// log.debug("Checking nc files.. ");
+//
+// for(WorkspaceItem item:folder.getChildren()){
+// try {
+// if(!item.isFolder()) {
+// String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
+// if(prefix.equals(".nc")){
+// // NC
+// PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken());
+// if(config.isProvidedMetadata()) {
+// String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
+// File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
+// if (meta!=null) req.setMetadata(meta);
+// }
+// server.put(req);
+// }
+// }
+// }catch(Exception e) {
+// try{
+// log.warn("Unabel to check item {} ",item.getPath(),e);
+// }catch(InternalErrorException e1) {
+// log.warn("Unabel to check item and to get Path {} ",item,e);
+// }
+// }
+// }
+// folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+""));
+// log.debug("Creating requests... ");
+// }
+//
+// private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException {
+// try{
+// WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId);
+// if(found==null) throw new ItemNotFoundException("Found item was null");
+// return NetUtils.toFile(((ExternalFile)found).getData());
+// }catch(ItemNotFoundException e) {
+// return null;
+// }
+// }
+//
+// // Remotely deletes folders which has history operations : RENAMING, REMOVAL
+// private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException {
+// List history=folder.getAccounting();
+// long lastUpdateTimeMillis=getLastUpdateTime(folder);
+// Date lastUpdate=new Date(lastUpdateTimeMillis);
+// log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate));
+//
+//
+// //look into history
+// boolean toDeleteCurrentFolder=false;
+// for(AccountingEntry entry: history) {
+// Date eventTime=entry.getDate().getTime();
+//
+// switch(entry.getEntryType()) {
+// case REMOVAL:
+//
+// case RENAMING:{
+// log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
+// if(eventTime.after(lastUpdate)) {
+// log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
+// toDeleteCurrentFolder=true;
+// }
+// }
+// }
+// if(toDeleteCurrentFolder) break;
+// }
+//
+//
+// //Delete Folder or scan children
+// if(toDeleteCurrentFolder) {
+// log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath);
+// try{
+// Commons.cleanupFolder(remoteFolderPath,targetToken);
+// }catch(Throwable t) {
+// log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t);
+// if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish
+// throw t;
+//
+// }
+// }else {
+// log.info("Folder is not to be cleaned up. Checking children..");
+// for(WorkspaceItem item:folder.getChildren())
+// if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName());
+// }
+// }
+//
+// private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException {
+// try{
+// return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME));
+// }catch(Throwable e) {
+// log.debug("Unable to get last update time for {}. Considering 0..",item.getPath());
+// return 0l;
+// }
+// }
+//
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java b/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java
index b8089d8..bb8b92e 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java
@@ -9,6 +9,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.SynchEngineImpl;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
+import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@@ -52,6 +53,6 @@ public interface SyncEngine {
public Set getAvailableCatalogsByToken(String token) throws InternalException;
-
+ public SyncEngineStatusDescriptor getStatus();
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java
index a740a09..4cfb568 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java
@@ -3,9 +3,6 @@ package org.gcube.usecases.ws.thredds.engine;
import java.io.File;
import java.util.HashSet;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
@@ -23,11 +20,14 @@ public class PublishRequest {
String name;
String id;
- public PublishItem(WorkspaceItem item) throws InternalErrorException {
- url=item.getPublicLink(false);
- name=item.getName();
- id=item.getId();
- }
+// public PublishItem(Item item) throws InternalErrorException {
+//
+//
+//
+// url=item.getPublicLink(false);
+// name=item.getName();
+// id=item.getId();
+// }
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java
index b9a53fe..01f020b 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java
@@ -1,17 +1,14 @@
package org.gcube.usecases.ws.thredds.engine;
-import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
-import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
-
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import java.net.URL;
import java.nio.file.Files;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
-import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
@@ -25,13 +22,7 @@ import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
-import org.gcube.resources.discovery.client.api.DiscoveryClient;
-import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
-import org.gcube.spatial.data.sdi.interfaces.Metadata;
-import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
-import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
-import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
import org.gcube.usecases.ws.thredds.Commons;
import org.gcube.usecases.ws.thredds.NetUtils;
@@ -90,7 +81,7 @@ public class PublishThread implements Runnable {
waitFor(request.getQueueId(), request.getQueueCount());
log.debug("Loading netcdfFile ..");
- File ncmlFile=NetUtils.download(request.getSource().getUrl());
+ File ncmlFile=NetUtils.download(new URL(request.getSource().getUrl()));
String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath()));
for(String reportId:request.getToGatherReportsId()) {
@@ -209,7 +200,7 @@ public class PublishThread implements Runnable {
log.debug("Alerting queue {}. Expected count is {} ",queueId,expected);
Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1));
sem.release();
- log.debug("Queue {} alerted. Remaining : {} out of {} ",queueId,sem.availablePermits(),expected);
+ log.debug(String.format("Queue %1$s alerted. Remaining : %2$s out of %3$s ",queueId,sem.availablePermits(),expected));
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java
index 81d1cfb..d3134d6 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java
@@ -11,9 +11,10 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status;
@@ -22,6 +23,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferToThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException;
+import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
@@ -37,7 +39,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Process {
-
+
private boolean submittedRequests=false;
private ProcessDescriptor descriptor;
@@ -48,7 +50,7 @@ public class Process {
private Queue queuedReports=new LinkedList<>();
-// private String folderId;
+ // private String folderId;
private WorkspaceFolderManager manager;
@@ -58,13 +60,13 @@ public class Process {
public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException {
log.debug("Created Process with id {} ",processId);
-// this.folderId=folderId;
+ // this.folderId=folderId;
manager=new WorkspaceFolderManager(folderId);
manager.lock(processId);
SynchFolderConfiguration folderConfig=manager.getSynchConfiguration();
try {
- descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().getPath(),System.currentTimeMillis(),processId,folderConfig);
+ descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().get().getPath(),System.currentTimeMillis(),processId,folderConfig);
}catch(Exception e) {
throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e);
}
@@ -75,10 +77,10 @@ public class Process {
}
- public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalErrorException, InternalException {
+ public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, StorageHubException, InternalException {
WorkspaceUtils.resetStatus(manager.getTheFolder());
status.setCurrentMessage("Analyzing folder..");
-
+
generateRequests(this,service, manager.getTheFolder());
submittedRequests=true;
if(status.getQueuedTransfers().get()>0) {
@@ -164,6 +166,7 @@ public class Process {
break;
}
}
+ status.setCurrentMessage("Waiting for requests ["+(status.getErrorCount().get()+status.getServedTransfers().get())+"/"+status.getQueuedTransfers().get()+"] to be served.");
status.getLogBuilder().append(
String.format("%s - item [%s] %s: %s \n", Constants.DATE_FORMAT.format(new Date(report.getCompletionTime())),
report.getElementName(),report.getStatus()+"",report.getMessage()));
@@ -203,27 +206,27 @@ public class Process {
}
- private static final void generateRequests(Process ownerProcess,ExecutorService service,WorkspaceFolder toScanFolder ) throws InternalErrorException, InternalException{
-
- String folderPath=toScanFolder.getPath();
-
+ private static final void generateRequests(Process ownerProcess,ExecutorService service,FolderContainer toScanFolder ) throws StorageHubException, InternalException{
+
+ String folderPath=toScanFolder.get().getPath();
+
log.info("Generating requests for folder {}",folderPath);
log.debug("Process is {} ",ownerProcess.getDescriptor());
Set handledWorkspaceItemEntries=new HashSet();
-
+
SynchFolderConfiguration config=ownerProcess.getDescriptor().getSynchConfiguration();
Set remoteChildrenNames;
Set localChildrenNames=new HashSet<>();
-
- List localFolderChildren=toScanFolder.getChildren();
- for(WorkspaceItem item:localFolderChildren) {
- localChildrenNames.add(item.getName());
+
+ List> localFolderChildren=toScanFolder.list().withAccounting().withMetadata().getContainers();
+ for(ItemContainer> item:localFolderChildren) {
+ localChildrenNames.add(item.get().getName());
}
-
- String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
+
+ String relativePath=toScanFolder.get().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken());
-
+
RemoteFileDescriptor folderDesc=null;
try{
folderDesc=folderController.getFileDescriptor();
@@ -232,41 +235,41 @@ public class Process {
folderController.createEmptyFolder(null);
folderDesc=folderController.getFileDescriptor();
}
-
+
remoteChildrenNames=new HashSet<>(folderDesc.getChildren());
-
-
+
+
//*********************** HANDLING ACCOUNTING ENTRIES
-
+
Set handledAccountingEntries=WorkspaceUtils.scanAccountingForStatus( toScanFolder, config, localChildrenNames, remoteChildrenNames, folderController, ownerProcess, service);
-
-
-
+
+
+
//SCAN FOLDER CONTENT
log.debug("Checking content of {} ",folderPath);
- for(WorkspaceItem item:localFolderChildren) {
+ for(ItemContainer> item:localFolderChildren) {
- if(item.isFolder()) {
+ if(item.getType().equals(ContainerType.FOLDER)) {
// RECURSIVE ON SUB FOLDERS
- generateRequests(ownerProcess,service,(WorkspaceFolder) item);
+ generateRequests(ownerProcess,service,(FolderContainer) item);
}else {
- Map props=item.getProperties().getProperties();
+ Map props=item.get().getMetadata().getMap();
String itemId=item.getId();
- String itemName=item.getName();
+ String itemName=item.get().getName();
// REQUESTS ARE EVALUATED ON PROPERTIES (SET BY PREVIOUS SCAN)
-
+
if(props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null)) {
try {
- SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS));
- log.trace("Found TBS item {}, name {}, status : ",item.getId(),item.getName(),status);
+ SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)+"");
+ log.trace(String.format("Found TBS item %1$s, name %2$s, status : %3$s",item.getId(),item.get().getName(),status));
SynchronizationRequest request=null;
switch(status) {
- case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder,item);
+ case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder.get(),item.get());
break;
- case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item, toScanFolder, null);
+ case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item.get(), toScanFolder.get(), null);
break;
}
@@ -277,34 +280,45 @@ public class Process {
}else log.debug("Item is up to date");
handledWorkspaceItemEntries.add(itemName);
}catch(Throwable t) {
- log.error("Unable to submit request for {} ID {} ",itemName,itemId,t);
+ log.error(String.format("Unable to submit request for %1$s ID %2$s ",itemName,itemId),t);
}
}
}
}
-
+
// check items to be imported
-
+
try {
Set toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, handledAccountingEntries, handledWorkspaceItemEntries, toScanFolder, folderController, config, ownerProcess, service);
log.debug("Checking if remote location contains folders to be imported...");
- for(String item:toImportItems) {
- if(folderController.getFileDescriptor(item).isDirectory()) {
- log.info("Creating folder {} under {} ",item,folderPath);
- try{
- WorkspaceFolder folder=toScanFolder.createFolder(item, "Imported from thredds");
+ for(String item:toImportItems) {
+ if(folderController.getFileDescriptor(item).isDirectory()) {
+
+ //check if folder already existing!!
+
+ FolderContainer folder=null;
+ try{
+ try {
+ ItemContainer> foundElement=WorkspaceUtils.scan(toScanFolder, item);
+ if(!foundElement.getType().equals(ContainerType.FOLDER))
+ throw new Exception("Conflict on WS item "+foundElement.getId()+". Remote element is a folder ");
+ }catch(ItemNotFoundException e) {
+ log.info("Creating folder {} under {} ",item,folderPath);
+ folder=toScanFolder.newFolder(item, "Imported from thredds");
WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId());
generateRequests(ownerProcess, service, folder);
- }catch(Throwable t) {
- log.error("Unable to import folder {} into {} ",item,folderPath);
}
+
+ }catch(Throwable t) {
+ log.error("Unable to import folder {} into {} ",item,folderPath);
}
}
+ }
}catch(InternalException e) {
log.error("Unable to check remote content with config {} ",config,e);
}
-
-
+
+
log.info("All requests for {} synchronization have been submitted [count {} ]. ",folderPath,ownerProcess.status.getQueuedTransfers().get());
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java
index 8095f97..86d371a 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java
@@ -7,7 +7,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -27,6 +26,7 @@ import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.CompletionCallback;
+import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@@ -53,15 +53,20 @@ public class SynchEngineImpl implements SyncEngine{
localProcesses=new ConcurrentHashMap<>();
+ // NB UNBOUNDED QUEUE MEANS ONLY CORE THREADS ARE EXECUTED
+
int scannerMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_MAX_SIZE));
int scannerCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_CORE_SIZE));
+// int scannerCoreSize=scannerMaxSize;
int scannerIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_IDLE_MS));
int transfersMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_MAX_SIZE));
int transfersCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_CORE_SIZE));
+// int transfersCoreSize=transfersMaxSize;
int transfersIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_IDLE_MS));
+
initializationExecutor= new ThreadPoolExecutor(scannerCoreSize, scannerMaxSize, scannerIdleMs,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
@@ -264,4 +269,10 @@ public class SynchEngineImpl implements SyncEngine{
return toReturn;
}
+
+ @Override
+ public SyncEngineStatusDescriptor getStatus() {
+ ThreadPoolExecutor exec=(ThreadPoolExecutor) synchronizationExecutor;
+ return new SyncEngineStatusDescriptor(exec.getActiveCount(), exec.getQueue().size(), LocalConfiguration.get().asMap());
+ }
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java
index a218a0b..90189b9 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java
@@ -7,6 +7,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
+import java.net.URL;
import java.nio.file.Files;
import java.util.Set;
@@ -16,7 +17,8 @@ import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import org.apache.tika.io.IOUtils;
+import org.apache.commons.io.IOUtils;
+import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
@@ -83,10 +85,10 @@ public class ThreddsController {
private void resetCallerToken() {
if(callerToken!=null) {
- log.trace("Resetting caller token {}. Target Token is {}, current is {} ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken()));
+ log.trace(String.format("Resetting caller token %1$s. Target Token is %2$s, current is %3$s ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())));
TokenSetter.setToken(callerToken);
callerToken=null;
- }else log.trace("Caller token {} already reset [current token {}]. Target Token is {}",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken));
+ }else log.trace(String.format("Caller token %1$s already reset [current token %2$s]. Target Token is %3$s",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)));
}
public final ThreddsInfo getThreddsInfo() {
@@ -208,6 +210,7 @@ public class ThreddsController {
public ThreddsCatalog createCatalog(String name) throws InternalException {
setTargetToken();
try{
+ SecurityTokenProvider.instance.get();
log.info("Creating catalog with name {} for path {} ",name,operatingPath);
String sdiUrl="http://"+getSDIServiceHost()+"/"+Constants.SDI_THREDDS_BASE_URL;
Response resp=getWebClient().target(sdiUrl).
@@ -267,7 +270,7 @@ public class ThreddsController {
}
- public TransferResult transferFile(Destination dest,String url,Set invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
+ public TransferResult transferFile(Destination dest,URL url,Set invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
setTargetToken();
try{DataTransferClient client=getDTClient(hostname);
if(invocations!=null&&!invocations.isEmpty())
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java
index e46cefc..dae4da4 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java
@@ -1,20 +1,17 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.io.File;
-import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.gcube.common.homelibrary.home.HomeLibrary;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Workspace;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.client.dsl.StorageHubClient;
+import org.gcube.common.storagehub.model.Metadata;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
import org.gcube.usecases.ws.thredds.Constants;
@@ -43,7 +40,7 @@ public class WorkspaceFolderManager {
}
- private WorkspaceFolder theFolder;
+ private FolderContainer theFolder;
private String folderId;
@@ -52,20 +49,21 @@ public class WorkspaceFolderManager {
private SynchFolderConfiguration config=null;
private ThreddsController threddsController=null;
- private Workspace ws;
+ private StorageHubClient sc;
public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException {
try{
- ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
- theFolder=(WorkspaceFolder) ws.getItem(folderId);
+// ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
+ sc=WorkspaceUtils.getClient();
+ theFolder=sc.open(folderId).asFolder();
this.folderId=folderId;
- }catch(WorkspaceException | InternalErrorException | HomeNotFoundException | UserNotFoundException e) {
+ }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to access folder id "+folderId,e);
}
}
- public WorkspaceFolder getTheFolder() {
+ public FolderContainer getTheFolder() {
return theFolder;
}
@@ -79,21 +77,17 @@ public class WorkspaceFolderManager {
private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
try {
- WorkspaceFolder root=(WorkspaceFolder) ws.getItem(getSynchConfiguration().getRootFolderId());
+ FolderContainer root=sc.open(getSynchConfiguration().getRootFolderId()).asFolder();
SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root);
return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken());
- }catch(WorkspaceException | InternalErrorException e) {
+ }catch(StorageHubException e) {
throw new WorkspaceInteractionException(e);
}
}
public boolean isRoot() throws WorkspaceNotSynchedException, WorkspaceInteractionException{
- try{
- return getSynchConfiguration().getRootFolderId().equals(theFolder.getId());
- }catch(InternalErrorException e) {
- throw new WorkspaceInteractionException(e);
- }
+ return getSynchConfiguration().getRootFolderId().equals(theFolder.getId());
}
@@ -103,20 +97,16 @@ public class WorkspaceFolderManager {
if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched.");
log.debug("Loading properties for ");
config=WorkspaceUtils.loadConfiguration(theFolder);
- }catch(InternalErrorException e) {
+ }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e);
}
}
return config;
}
- public boolean isSynched() throws WorkspaceInteractionException {
- try{
- Map props=theFolder.getProperties().getProperties();
+ public boolean isSynched(){
+ Map props=theFolder.get().getMetadata().getMap();
return props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null);
- }catch(InternalErrorException e) {
- throw new WorkspaceInteractionException("Unable to check Synch flag on "+folderId,e);
- }
}
@@ -126,9 +116,9 @@ public class WorkspaceFolderManager {
SynchFolderConfiguration config=getSynchConfiguration();
try{
- checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder));
- return new SyncFolderDescriptor(this.folderId,this.theFolder.getPath(),config);
- }catch(InternalErrorException e) {
+ checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder.get()));
+ return new SyncFolderDescriptor(this.folderId,this.theFolder.get().getPath(),config);
+ }catch(StorageHubException e) {
throw new WorkspaceInteractionException(e);
}
}
@@ -188,7 +178,7 @@ public class WorkspaceFolderManager {
}catch(InternalException e) {
throw new InternalException ("Unable to check/initialize remote folder",e);
- }catch(InternalErrorException e) {
+ }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to set Properties to "+folderId,e);
}
}
@@ -202,12 +192,12 @@ public class WorkspaceFolderManager {
WorkspaceUtils.cleanItem(theFolder);
if(deleteRemote)
getThreddsController().createEmptyFolder(null);
- }catch(InternalErrorException e) {
+ }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to cleanup "+folderId,e);
}
}
- public void setLastUpdateTime() throws InternalErrorException {
+ public void setLastUpdateTime() throws StorageHubException {
WorkspaceUtils.setLastUpdateTime(theFolder, System.currentTimeMillis());
}
@@ -251,12 +241,12 @@ public class WorkspaceFolderManager {
- private static void checkFolder(WorkspaceFolder folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws InternalErrorException, InternalException {
+ private static void checkFolder(FolderContainer folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws StorageHubException, InternalException {
// Check folder configuration
- log.trace("Checking folder {} ",folder.getPath());
+ log.trace("Checking folder {} ",folder.get().getPath());
log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder);
- String folderName=folder.getName();
+ String folderName=folder.get().getName();
String currentRemotePath=rootConfig.getRemotePath()+((relativePathFromRootFolder==null)?"":"/"+relativePathFromRootFolder);
@@ -270,19 +260,19 @@ public class WorkspaceFolderManager {
log.debug("Initializing properties for {} ",folderName);
//INIT PROPERTIES IF NOT PRESENT
- if(!WorkspaceUtils.isConfigured(folder))
+ if(!WorkspaceUtils.isConfigured(folder.get()))
WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
- for(WorkspaceItem item:folder.getChildren()) {
- String itemName=item.getName();
+ for(ItemContainer> item:folder.list().withAccounting().withMetadata().getContainers()) {
+ String itemName=item.get().getName();
String itemRelativePath=(relativePathFromRootFolder==null)?itemName:relativePathFromRootFolder+"/"+itemName;
String itemRemotePath=currentRemotePath+"/"+itemName;
- if(item.isFolder()) {
+ if(item.getType().equals(ContainerType.FOLDER)) {
if(recursive)
- checkFolder((WorkspaceFolder) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine);
+ checkFolder((FolderContainer) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine);
else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
}else if(rootConfig.matchesFilter(itemName)) {
- if(!WorkspaceUtils.isConfigured(item))
+ if(!WorkspaceUtils.isConfigured(item.get()))
WorkspaceUtils.initProperties(item, null, null, null,null,null,null);
}
currentFolderExistingItem.add(itemName);
@@ -305,31 +295,50 @@ public class WorkspaceFolderManager {
currentFolderExistingItem, remoteFolderItems,
controller, null, null);
if(accountingEntries.isEmpty()) {
- log.debug("No accounting entries found");
+ log.debug("No Accounting Entries to be managed..");
folderStatus=SynchronizationStatus.UP_TO_DATE;
}
// CHECK WS ITEMS
- for(WorkspaceItem item:folder.getChildren())
- if(item.isFolder()||rootConfig.matchesFilter(item.getName())) {
- SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item, remoteFolderItems, controller,lastUpdatedRoutine);
- item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+""));
+ for(ItemContainer> item:folder.list().withAccounting().withMetadata().getContainers())
+ if(item.getType().equals(ContainerType.FOLDER)||rootConfig.matchesFilter(item.get().getName())) {
+ SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item.get(), remoteFolderItems, controller,lastUpdatedRoutine);
+
+ Metadata meta=item.get().getMetadata();
+ Map map=meta.getMap();
+ map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+"");
+ meta.setMap(map);
+ item.setMetadata(meta);
folderStatus=folderStatus.equals(SynchronizationStatus.UP_TO_DATE)?itemStatus:folderStatus;
- }
+ }
// CHECK REMOTE FOLDER
if(folderStatus.equals(SynchronizationStatus.UP_TO_DATE)) {
Set toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, accountingEntries, currentFolderExistingItem, folder, controller, rootConfig, null, null);
if(!toImportItems.isEmpty()) folderStatus=SynchronizationStatus.OUTDATED_WS;
}
-
- folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+""));
+
+ Metadata meta=folder.get().getMetadata();
+ Map map=meta.getMap();
+ map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+"");
+ meta.setMap(map);
+ folder.setMetadata(meta);
}else {
// Remote Folder not existing, set everything to OUTDATED_REMOTE
- for(WorkspaceItem item:folder.getChildren())
- item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""));
- folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""));
+ for(ItemContainer> item:folder.list().withMetadata().getContainers()) {
+ Metadata meta=item.get().getMetadata();
+ Map map=meta.getMap();
+ map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"");
+ meta.setMap(map);
+ item.setMetadata(meta);
+ }
+
+ Metadata meta=folder.get().getMetadata();
+ Map map=meta.getMap();
+ map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"");
+ meta.setMap(map);
+ folder.setMetadata(meta);
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java
index 85bd453..f255150 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java
@@ -1,26 +1,31 @@
package org.gcube.usecases.ws.thredds.engine.impl;
-import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
-import org.gcube.common.homelibary.model.items.type.WorkspaceItemType;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.workspace.Properties;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
-import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRemoval;
-import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRenaming;
+
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.client.dsl.StorageHubClient;
+import org.gcube.common.storagehub.model.Metadata;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
+import org.gcube.common.storagehub.model.items.FolderItem;
+import org.gcube.common.storagehub.model.items.Item;
+import org.gcube.common.storagehub.model.items.nodes.accounting.AccountEntry;
+import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRemoval;
+import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRenaming;
+import org.gcube.common.storagehub.model.types.WorkspaceItemType;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException;
+import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@@ -33,10 +38,12 @@ public class WorkspaceUtils {
-
+ public static StorageHubClient getClient() throws StorageHubException{
+ return new StorageHubClient();
+ }
/**
- * Checks current accounting info in order to infere synchronization status.
+ * Checks current accounting info in order to infer synchronization status.
* OwnerProcess and service parameters can be null for check purposes.
*
* @param folderPath
@@ -52,53 +59,62 @@ public class WorkspaceUtils {
* @throws InternalErrorException
*/
static Set scanAccountingForStatus(
- WorkspaceFolder toScanFolder,
+ FolderContainer toScanFolder,
SynchFolderConfiguration config,
Set localChildrenNames,
Set remoteChildrenNames,
ThreddsController folderController,
Process ownerProcess,
- ExecutorService service) throws InternalErrorException{
+ ExecutorService service) throws StorageHubException{
Set handledAccountingEntries=new HashSet<>();
- log.debug("Checking history of {} ",toScanFolder.getPath());
+ FolderItem toScanFolderItem=toScanFolder.get();
+
+
+ log.debug("Checking history of {} ",toScanFolderItem.getPath());
+
+
- String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
+ String relativePath=toScanFolderItem.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
Date folderLastUpdateTime=null;
try{
- folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolder);
+ folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolderItem);
}catch(Throwable t) {
- log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolder.getName(),t);
+ log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolderItem.getName(),t);
folderLastUpdateTime=new Date(0l);
}
// scanning for deletions
- log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolder.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime));
- for(AccountingEntry entry:toScanFolder.getAccounting()) {
+ log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolderItem.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime));
+ for(AccountEntry entry:toScanFolderItem.getAccounting().getEntries()) {
try {
Date eventTime=entry.getDate().getTime();
if(folderLastUpdateTime==null|| eventTime.after(folderLastUpdateTime)) { // SKIP IF ENTRY OLDER THAN LAST UPDATE TIME
String toDeleteRemote=null;
- switch(entry.getEntryType()) {
+ switch(entry.getType()) {
case CUT:
case REMOVAL:{
- AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry;
- if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER)||
+ AccountFolderEntryRemoval removalEntry=(AccountFolderEntryRemoval) entry;
+ if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER.toString())||
config.matchesFilter(removalEntry.getItemName()))
toDeleteRemote=removalEntry.getItemName();
break;
}
- case RENAMING:{
- AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry;
- WorkspaceItem newItem=toScanFolder.find(renamingEntry.getNewItemName());
- if(newItem.isFolder()||config.matchesFilter(renamingEntry.getOldItemName()))
+ case RENAMING:{
+ AccountFolderEntryRenaming renamingEntry=(AccountFolderEntryRenaming) entry;
+ ItemContainer> newItem=toScanFolder.findByName(renamingEntry.getNewItemName()).getContainers().get(0);
+ if(newItem.getType().equals(ContainerType.FOLDER)||config.matchesFilter(renamingEntry.getOldItemName()))
toDeleteRemote=renamingEntry.getOldItemName();
break;
- }
+ }
+
+ default : {
+ log.debug("Skpping accounting entry {} ",entry.getType());
+ }
}
@@ -117,7 +133,7 @@ public class WorkspaceUtils {
if(service!=null) {
log.debug("Service is not null. Submitting request ... ");
if(eventTime.after(remoteDate)) {
- service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolder,toDeleteRemote)));
+ service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolderItem,toDeleteRemote)));
handledAccountingEntries.add(toDeleteRemote);
log.debug("Submitted DELETION request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
}
@@ -161,13 +177,15 @@ public class WorkspaceUtils {
RemoteFileDescriptor folderDesc,
Set handledAccountingEntries,
Set handledWorkspaceItemEntries,
- WorkspaceFolder toScanFolder,
+ FolderContainer toScanFolder,
ThreddsController folderController,
SynchFolderConfiguration config,
Process ownerProcess,
- ExecutorService service) throws InternalException, InternalErrorException{
+ ExecutorService service) throws InternalException, StorageHubException{
- log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanFolder.getPath(),folderDesc.getAbsolutePath());
+ FolderItem toScanItem=toScanFolder.get();
+
+ log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanItem.getPath(),folderDesc.getAbsolutePath());
Set handledRemoteElements=new HashSet();
// String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
@@ -189,7 +207,7 @@ public class WorkspaceUtils {
log.debug("Child {} matches filter...");
handledRemoteElements.add(child);
if(service!=null) {
- service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, child)));
+ service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanItem, child)));
log.debug("Submitted IMPORT request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
}
// import if matching
@@ -201,10 +219,12 @@ public class WorkspaceUtils {
}
- static void initProperties(WorkspaceItem toInit, String remotePath, String filter, String targetToken,
- String catalogName,Boolean validateMeta, String rootFolderId) throws InternalErrorException {
+ static void initProperties(ItemContainer> toInit, String remotePath, String filter, String targetToken,
+ String catalogName,Boolean validateMeta, String rootFolderId) throws StorageHubException {
- Map toSetProperties=toInit.getProperties().getProperties();
+
+ Metadata meta=toInit.get().getMetadata();
+ Map toSetProperties=meta.getMap();
initIfMissing(toSetProperties,Constants.WorkspaceProperties.TBS,"true");
@@ -212,7 +232,7 @@ public class WorkspaceUtils {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_STATUS,StepReport.Status.OK+"");
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.UP_TO_DATE+"");
- if(toInit.isFolder()) {
+ if(toInit.getType().equals(ContainerType.FOLDER)) {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE);
@@ -222,26 +242,28 @@ public class WorkspaceUtils {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId);
}else {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.METADATA_UUID,null);
- }
- toInit.getProperties().addProperties(toSetProperties);
+ }
+ meta.setMap(toSetProperties);
+ toInit.setMetadata(meta);
+
}
- private static void initIfMissing(Map current,String key,String defaultValue) {
+ private static void initIfMissing(Map current,String key,String defaultValue) {
if(!current.containsKey(key)||
current.get(key)==null||
current.get(key).equals("null")) current.put(key, defaultValue);
}
- static boolean isConfigured(WorkspaceItem toCheck) throws InternalErrorException {
- return isConfigured(toCheck.getProperties().getProperties());
+ static boolean isConfigured(Item toCheck) throws StorageHubException {
+ return isConfigured(toCheck.getMetadata().getMap());
}
- static boolean isConfigured(Map toCheckProperties) {
+ static boolean isConfigured(Map toCheckProperties) {
return (toCheckProperties.containsKey(Constants.WorkspaceProperties.TBS)&&toCheckProperties.get(Constants.WorkspaceProperties.TBS)!=null);
}
- static SynchronizationStatus getStatusAgainstRemote(WorkspaceItem item, Set existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException, InternalErrorException, RemoteFileNotFoundException {
+ static SynchronizationStatus getStatusAgainstRemote(Item item, Set existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException,RemoteFileNotFoundException {
String itemName=item.getName();
SynchronizationStatus status=SynchronizationStatus.OUTDATED_REMOTE;
if(existingRemote.contains(itemName)) {
@@ -269,88 +291,132 @@ public class WorkspaceUtils {
}
- //
- // /**
- // *
- // * @return max date between creation time, last modification time && LAST-UPDATE-PROP
- // * @throws InternalErrorException
- // * @throws NumberFormatException
- // */
- // static Date getMaxLastUpdate(WorkspaceItem item) throws NumberFormatException, InternalErrorException {
- // return new Date(Long.max(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)),item.getLastModificationTime().getTimeInMillis()));
- // }
- static Date safelyGetLastUpdate(WorkspaceItem item) throws InternalErrorException {
+ static Date safelyGetLastUpdate(Item item){
try {
- return new Date(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)));
+ return new Date(Long.parseLong(item.getMetadata().getMap().get(Constants.WorkspaceProperties.LAST_UPDATE_TIME)+""));
}catch(NumberFormatException e) {
log.debug("Unable to get last update time for {} ",item.getName(),e);
return new Date(0l);
}
}
- public static boolean isModifiedAfter(WorkspaceItem item,Date fromDate) throws InternalErrorException {
- for(AccountingEntry entry:item.getAccounting()) {
- if(entry.getDate().getTime().after(fromDate)) {
- switch(entry.getEntryType()) {
- case PASTE:
- case CREATE:
- case RESTORE:
- case UPDATE:
- case ADD: return true;
- }
- }
- }
- return false;
+ public static boolean isModifiedAfter(Item item,Date fromDate) {
+ return item.getLastModificationTime().after(fromDate);
+// for(AccountingEntry entry:item.getAccounting()) {
+// if(entry.getDate().getTime().after(fromDate)) {
+// switch(entry.getEntryType()) {
+// case PASTE:
+// case CREATE:
+// case RESTORE:
+// case UPDATE:
+// case ADD: return true;
+// }
+// }
+// }
+// return false;
}
- static void cleanItem(WorkspaceItem item) throws InternalErrorException {
- Properties props=item.getProperties();
- if(props.hasProperty(Constants.WorkspaceProperties.TBS)) {
- if(item.isFolder()) {
- props.addProperties(Constants.cleanedFolderPropertiesMap);
- for(WorkspaceItem child : ((WorkspaceFolder)item).getChildren())
+ static void cleanItem(ItemContainer> itemContainer) throws StorageHubException{
+ Metadata meta=itemContainer.get().getMetadata();
+ Map map=meta.getMap();
+ if(map.containsKey(Constants.WorkspaceProperties.TBS)) {
+ if(itemContainer.getType().equals(ContainerType.FOLDER)) {
+ map=Constants.cleanedFolderPropertiesMap;
+ for(ItemContainer> child : ((FolderContainer)itemContainer).list().withMetadata().getContainers())
cleanItem(child);
- }else props.addProperties(Constants.cleanedItemPropertiesMap);
+ }else map=Constants.cleanedItemPropertiesMap;
+
+ //Actually posting cleaned map
+ meta.setMap(map);
+ log.debug("Setting meta to item "+itemContainer.getId()+" : "+itemContainer.get().getPath());
+ itemContainer.setMetadata(meta);
}
}
- static void setLastUpdateTime(WorkspaceFolder folder,long toSetTime) throws InternalErrorException {
- StepReport.Status currentWSStatus=StepReport.Status.valueOf(folder.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_STATUS));
+ static void setLastUpdateTime(FolderContainer folder,long toSetTime) throws StorageHubException {
+
+ Metadata meta=folder.get().getMetadata();
+ Map map=meta.getMap();
+
+ StepReport.Status currentWSStatus=StepReport.Status.valueOf(map.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+"");
if(currentWSStatus.equals(StepReport.Status.OK))
- folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+""));
+ map.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+"");
- for(WorkspaceItem item:folder.getChildren())
- if(item.isFolder()) setLastUpdateTime((WorkspaceFolder) item, toSetTime);
+ //Actually posting cleaned map
+ meta.setMap(map);
+ folder.setMetadata(meta);
+
+// for(ItemContainer> child : folder.list().withMetadata().getContainers())
+// if(child.getType().equals(ContainerType.FOLDER))
+// setLastUpdateTime((FolderContainer) child, toSetTime);
}
- public static SynchFolderConfiguration loadConfiguration(WorkspaceItem item) throws InternalErrorException {
- if(item.isFolder()) {
- Properties props=item.getProperties();
+ public static SynchFolderConfiguration loadConfiguration(ItemContainer> item) throws StorageHubException {
+ if(item.getType().equals(ContainerType.FOLDER)) {
+ Map map=item.get().getMetadata().getMap();
SynchFolderConfiguration config=new SynchFolderConfiguration();
- config.setFilter(props.getPropertyValue(Constants.WorkspaceProperties.SYNCH_FILTER));
- config.setRemotePath(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH));
- config.setRemotePersistence(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
- config.setTargetToken(props.getPropertyValue(Constants.WorkspaceProperties.TARGET_TOKEN));
- config.setToCreateCatalogName(props.getPropertyValue(Constants.WorkspaceProperties.RELATED_CATALOG));
- config.setValidateMetadata(Boolean.parseBoolean(props.getPropertyValue(Constants.WorkspaceProperties.VALIDATE_METADATA)));
- config.setRootFolderId(props.getPropertyValue(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
+ config.setFilter(""+map.get(Constants.WorkspaceProperties.SYNCH_FILTER));
+ config.setRemotePath(""+map.get(Constants.WorkspaceProperties.REMOTE_PATH));
+ config.setRemotePersistence(""+map.get(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
+ config.setTargetToken(""+map.get(Constants.WorkspaceProperties.TARGET_TOKEN));
+ config.setToCreateCatalogName(""+map.get(Constants.WorkspaceProperties.RELATED_CATALOG));
+ config.setValidateMetadata(Boolean.parseBoolean(""+map.get(Constants.WorkspaceProperties.VALIDATE_METADATA)));
+ config.setRootFolderId(""+map.get(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
return config;
- }else return loadConfiguration(item.getParent());
+ }else {
+ FolderContainer parentFolder=getClient().open(item.get().getParentId()).asFolder();
+
+ return loadConfiguration(parentFolder);
+ }
}
- static void resetStatus(WorkspaceItem item) throws InternalErrorException {
- if(item.isFolder()) {
- for(WorkspaceItem child: ((WorkspaceFolder)item).getChildren())
+ static void resetStatus(ItemContainer> item) throws StorageHubException {
+ if(item.getType().equals(ContainerType.FOLDER)) {
+ for(ItemContainer> child : ((FolderContainer)item).list().withMetadata().getContainers())
resetStatus(child);
}
- Map props=item.getProperties().getProperties();
- if(props.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) {
- props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+"");
- item.getProperties().addProperties(props);
+ Metadata meta=item.get().getMetadata();
+ Map map=meta.getMap();
+ if(map.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) {
+ map.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+"");
+ //Actually posting cleaned map
+ meta.setMap(map);
+ item.setMetadata(meta);
}
}
+
+ public static void addParameters(ItemContainer> item, Map toAdd) throws StorageHubException {
+ Metadata meta=item.get().getMetadata();
+ Map current=meta.getMap();
+ current.putAll(toAdd);
+ meta.setMap(current);
+ item.setMetadata(meta);
+ }
+
+ public static void addParameter(ItemContainer> item, String key, Object value) throws StorageHubException {
+ Metadata meta=item.get().getMetadata();
+ Map current=meta.getMap();
+ current.put(key,value);
+ meta.setMap(current);
+ item.setMetadata(meta);
+ }
+
+
+ public static ItemContainer> scan(FolderContainer folder,String path) throws ItemNotFoundException, StorageHubException{
+ String toLookFor=path.substring((path.startsWith("/")?1:0), path.length());
+ String[] split=toLookFor.split("/");
+ toLookFor=split[0];
+ for(ItemContainer> item:folder.list().withMetadata().getContainers())
+ if(item.get().getName().equals(toLookFor)) {
+ if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length()));
+ else return item;
+ }
+ throw new ItemNotFoundException("Unable to find "+path);
+ }
+
+
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java
index e82f29b..48952f5 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java
@@ -1,6 +1,6 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
+import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
@@ -10,7 +10,7 @@ public class DeleteRemoteRequest extends SynchronizationRequest {
private String toRemoveName;
- public DeleteRemoteRequest(Process process,WorkspaceFolder location,String name) {
+ public DeleteRemoteRequest(Process process,FolderItem location,String name) {
super(process,location);
this.toRemoveName=name;
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java
index bcb63a7..c0b5e1a 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java
@@ -2,7 +2,7 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.util.concurrent.ExecutorService;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
@@ -45,7 +45,7 @@ public class ProcessInitializationThread implements Runnable {
} catch (InternalException e) {
log.error("Unable to proceed..",e);
theProcess.cancel();
- } catch (InternalErrorException e) {
+ } catch (StorageHubException e) {
log.error("Unable to proceed..",e);
theProcess.cancel();
}catch(Throwable t) {
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java
index f277ca0..31fcca6 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java
@@ -3,10 +3,12 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.io.BufferedWriter;
import java.io.FileWriter;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
+import org.gcube.common.storagehub.model.items.FolderItem;
+import org.gcube.common.storagehub.model.items.Item;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.SyncEngine;
+import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@@ -81,33 +83,30 @@ public class RequestLogger {
}
- private static final String getRemotePath(WorkspaceItem item) {
+ private static final String getRemotePath(Item item) {
try{
- if(item.isFolder()) return item.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
- else return getRemotePath(item.getParent());
- }catch(InternalErrorException e) {
+ if(item instanceof FolderItem) return item.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
+ else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get());
+ }catch(StorageHubException e) {
log.warn("Unable to get Remote Path ",e);
return "N/A";
}
}
- private static final String getName(WorkspaceItem item) {
- try {
+ private static final String getName(Item item) {
return item.getName();
- }catch(InternalErrorException e) {
- log.warn("Unable to get name ",e);
- return "N/A";
- }
}
- private static final String getWSPath(WorkspaceItem item) {
+ private static final String getWSPath(Item item) {
try{
- if(item.isFolder()) return item.getPath();
- else return getRemotePath(item.getParent());
- }catch(InternalErrorException e) {
+ if(item instanceof FolderItem) return item.getPath();
+ else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get());
+ }catch(StorageHubException e) {
log.warn("Unable to get WS Path ",e);
return "N/A";
}
}
+
+
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java
index 40c9d16..f3a95ca 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java
@@ -1,6 +1,6 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
+import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
@@ -12,5 +12,5 @@ public abstract class SynchronizationRequest {
@NonNull
private Process process;
@NonNull
- private WorkspaceFolder location;
+ private FolderItem location;
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java
index 8bd2dd1..d79f756 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java
@@ -8,9 +8,15 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FileContainer;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.client.dsl.StorageHubClient;
+import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
+import org.gcube.common.storagehub.model.items.FolderItem;
+import org.gcube.common.storagehub.model.items.Item;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
@@ -32,7 +38,6 @@ import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.CancellationException;
import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError;
-import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.StepReport.OperationType;
@@ -60,22 +65,25 @@ public class SynchronizationThread implements Runnable {
RequestLogger.get().log(theRequest);
String reportMessage="Never started";
String reportItemName="Still Unknown";
- Status toSetStatus=Status.ERROR;
- WorkspaceFolder parentFolder=theRequest.getLocation();
+ Status toSetStatus=Status.ERROR;
+
+ FolderItem parentFolderItem=theRequest.getLocation();
try {
+ StorageHubClient client=WorkspaceUtils.getClient();
+ FolderContainer parentFolder=client.open(parentFolderItem.getId()).asFolder();
checkCancelledProcess();
- SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(theRequest.getLocation());
+ SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(parentFolder);
ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken());
if(theRequest instanceof TransferToThreddsRequest) {
TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest;
- WorkspaceItem item=request.getToTransfer();
-
+ Item item=request.getToTransfer();
+ FileContainer itemContainer=client.open(item.getId()).asFile();
//look for metadata in same folder
String itemName=item.getName();
reportItemName=itemName;
String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml";
- WorkspaceItem metadataItem=getFileByName(item.getParent(),false,toLookMetadataName);
+ FileContainer metadataItem=getFileByName(client.open(item.getParentId()).asFolder(),false,toLookMetadataName);
// if not present, generate with sis/geotk
Destination toSetDestination=new Destination();
@@ -86,7 +94,7 @@ public class SynchronizationThread implements Runnable {
toSetDestination.setPersistenceId(synchConfig.getRemotePersistence());
//NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
- String fileLocation=request.getLocation().getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
+ String fileLocation=request.getLocation().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
toSetDestination.setSubFolder(fileLocation);
@@ -108,17 +116,17 @@ public class SynchronizationThread implements Runnable {
checkCancelledProcess();
- TransferResult result=controller.transferFile(toSetDestination, item.getPublicLink(false), invocations);
+ TransferResult result=controller.transferFile(toSetDestination, itemContainer.getPublicLink(), invocations);
- Map toSetProperties=new HashMap();
+ Map toSetProperties=new HashMap();
String toSetMetadataUUID=null;
Boolean validateMetadata=synchConfig.getValidateMetadata();
-
-
+
+
checkCancelledProcess();
if(metadataItem==null) {
ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID);
@@ -134,15 +142,18 @@ public class SynchronizationThread implements Runnable {
toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID);
toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+"");
- item.getProperties().addProperties(toSetProperties);
+ org.gcube.common.storagehub.model.Metadata meta=item.getMetadata();
+ meta.setMap(toSetProperties);
+ item.setMetadata(meta);
+
reportMessage="Successfully transferred and published.";
// End ws->th
} else if(theRequest instanceof TransferFromThreddsRequest) {
- Map toSetProperties=new HashMap();
+ Map toSetProperties=new HashMap();
TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest;
String toImportName=null;
- WorkspaceItem targetItem=null;
+ FileContainer targetItem=null;
if(importRequest.getTargetItem()==null) {
//Target Item will be created
toImportName=importRequest.getRemoteFilename();
@@ -150,29 +161,38 @@ public class SynchronizationThread implements Runnable {
}else {
//Target Item already exists
toImportName=importRequest.getTargetItem().getName();
- targetItem=importRequest.getTargetItem();
+ targetItem=client.open(importRequest.getTargetItem().getId()).asFile();
reportMessage="Updating file..";
}
reportItemName=toImportName;
-
+
RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName);
toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+"");
InputStream source=null;
try {
source=controller.getInputStream(toImportName);
- if(targetItem==null)
- targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source);
- else
- targetItem.updateItem(source);
- targetItem.getProperties().addProperties(toSetProperties);
+ targetItem=parentFolder.uploadFile(source, toImportName, "Imported from Thredds");
+ // if(targetItem==null)
+ // targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source);
+ // else {
+ // targetItem.updateItem(source);
+ // }
+
+
+ org.gcube.common.storagehub.model.Metadata meta=targetItem.get().getMetadata();
+ meta.setMap(toSetProperties);
+ targetItem.setMetadata(meta);
}finally {
if(source!=null)
source.close();
}
reportMessage="File successfully imported";
-
+
+
+
+
}else if(theRequest instanceof DeleteRemoteRequest) {
DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest;
reportItemName=deleteRequest.getToRemoveName();
@@ -201,7 +221,7 @@ public class SynchronizationThread implements Runnable {
log.debug("Remote File not found ",e);
reportMessage="Remote File not found : "+e.getMessage();
toSetStatus=Status.ERROR;
- }catch(InternalErrorException e) {
+ }catch(StorageHubException e) {
log.debug("Internal generic exception ",e);
reportMessage="Internal error : "+e.getMessage();
toSetStatus=Status.ERROR;
@@ -210,7 +230,7 @@ public class SynchronizationThread implements Runnable {
reportMessage="Unexpected exception : "+t.getMessage();
toSetStatus=Status.ERROR;
}finally {
- updateParentProperty(parentFolder, toSetStatus);
+ updateParentProperty(parentFolderItem, toSetStatus);
submitReport(reportItemName,reportMessage,toSetStatus);
ProcessIdProvider.instance.reset();
}
@@ -218,19 +238,32 @@ public class SynchronizationThread implements Runnable {
@Synchronized
- private static void updateParentProperty(WorkspaceFolder folder,StepReport.Status toSetStatus) {
- try {
- String currentValue=folder.getProperties().getProperties().get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS);
- if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null"))
- folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""));
- else {
- StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue);
- if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus))
- folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""));
+ private static void updateParentProperty(FolderItem folderItem,StepReport.Status toSetStatus) {
+ boolean retry=false;
+ do {
+ try {
+ retry=false;
+ FolderContainer folder=WorkspaceUtils.getClient().open(folderItem.getId()).asFolder();
+ org.gcube.common.storagehub.model.Metadata folderMeta=folder.get().getMetadata();
+ Map props=folderMeta.getMap();
+ String currentValue=props.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+"";
+ if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null"))
+ props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
+ else {
+ StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue);
+ if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus))
+ props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
+ }
+
+ folderMeta.setMap(props);
+ folder.setMetadata(folderMeta);
+ }catch(ItemLockedException e) {
+ log.debug("Item locked retrying.. ",e);
+ retry=true;
+ }catch(Throwable t) {
+ log.warn("Unable to update folder status ",t);
}
- }catch(Throwable t) {
- log.warn("Unable to update folder status ",t);
- }
+ }while(retry);
}
@@ -254,11 +287,11 @@ public class SynchronizationThread implements Runnable {
- private static MetadataReport publishMetadata(WorkspaceItem toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{
+ private static MetadataReport publishMetadata(FileContainer toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{
File tempMetaFile=null;
try {
Metadata meta=SDIAbstractPlugin.metadata().build();
- tempMetaFile=NetUtils.download(toPublish.getPublicLink(false));
+ tempMetaFile=NetUtils.download(toPublish.getPublicLink());
log.debug("Publishing metadata {} ",filename);
@@ -274,14 +307,14 @@ public class SynchronizationThread implements Runnable {
}
- private static final WorkspaceItem getFileByName(WorkspaceFolder toLookIntoFolder,boolean recursive,String toLookForName) throws InternalErrorException {
- log.debug("Looking for {} into {} [recursive {} ]",toLookForName,toLookIntoFolder.getPath()+" ID "+toLookIntoFolder.getId(),recursive);
- for(WorkspaceItem item : toLookIntoFolder.getChildren())
- if(!item.isFolder()&&item.getName().equals(toLookForName)) return item;
+ private static final FileContainer getFileByName(FolderContainer toLookIntoFolder,boolean recursive,String toLookForName) throws StorageHubException {
+ log.debug(String.format("Looking for %1$s into %2$s [recursive %3$s ]",toLookForName,toLookIntoFolder.get().getPath()+" ID "+toLookIntoFolder.getId(),recursive));
+ for(ItemContainer> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
+ if(!item.getType().equals(ContainerType.FOLDER)&&item.get().getName().equals(toLookForName)) return (FileContainer) item;
if(recursive) {
- for(WorkspaceItem item : toLookIntoFolder.getChildren())
- if(item.isFolder()) return getFileByName((WorkspaceFolder) item, recursive, toLookForName);
+ for(ItemContainer> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
+ if(item.getType().equals(ContainerType.FOLDER)) return getFileByName((FolderContainer) item, recursive, toLookForName);
}
return null;
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java
index 160cfd7..57093c6 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java
@@ -1,23 +1,32 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
+import org.gcube.common.storagehub.model.items.FolderItem;
+import org.gcube.common.storagehub.model.items.Item;
+import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
@Data
+@Slf4j
public class TransferFromThreddsRequest extends SynchronizationRequest {
- private WorkspaceItem targetItem; // can be null
+ private Item targetItem; // can be null
private String remoteFilename;
- public TransferFromThreddsRequest(Process process, WorkspaceItem targetItem, WorkspaceFolder containingFolder,
+ public TransferFromThreddsRequest(Process process, Item targetItem, FolderItem containingFolder,
String remoteFilename) {
super(process,containingFolder);
this.targetItem = targetItem;
this.remoteFilename = remoteFilename;
+ if(targetItem==null)
+ log.debug(String.format("Created IMPORT request of %1$s into %1$s from %3$s",
+ remoteFilename,containingFolder.getPath(),containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
+ else
+ log.debug(String.format("Created UPDATE %1$s from %2$s", targetItem.getPath(),
+ containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java
index 42bbcaf..ef8d17b 100644
--- a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java
+++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java
@@ -1,23 +1,27 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
+import org.gcube.common.storagehub.model.items.FolderItem;
+import org.gcube.common.storagehub.model.items.Item;
+import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Getter;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
@Getter
@Setter
-
+@Slf4j
public class TransferToThreddsRequest extends SynchronizationRequest {
- private WorkspaceItem toTransfer;
+ private Item toTransfer;
- public TransferToThreddsRequest(Process process,WorkspaceFolder location, WorkspaceItem toTransfer) {
+ public TransferToThreddsRequest(Process process,FolderItem location, Item toTransfer) {
super(process,location);
this.toTransfer = toTransfer;
+ log.debug(String.format("Created SEND request from $1%s to $2%s",
+ toTransfer.getPath(),location.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
}
}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java
new file mode 100644
index 0000000..25efa86
--- /dev/null
+++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java
@@ -0,0 +1,35 @@
+package org.gcube.usecases.ws.thredds.faults;
+
+public class ItemNotFoundException extends WorkspaceInteractionException {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = -6818076224344812668L;
+
+ public ItemNotFoundException() {
+ // TODO Auto-generated constructor stub
+ }
+
+ public ItemNotFoundException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ // TODO Auto-generated constructor stub
+ }
+
+ public ItemNotFoundException(String message, Throwable cause) {
+ super(message, cause);
+ // TODO Auto-generated constructor stub
+ }
+
+ public ItemNotFoundException(String message) {
+ super(message);
+ // TODO Auto-generated constructor stub
+ }
+
+ public ItemNotFoundException(Throwable cause) {
+ super(cause);
+ // TODO Auto-generated constructor stub
+ }
+
+}
diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java
new file mode 100644
index 0000000..ce1c04e
--- /dev/null
+++ b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java
@@ -0,0 +1,18 @@
+package org.gcube.usecases.ws.thredds.model;
+
+import java.util.Map;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.ToString;
+
+@Getter
+@AllArgsConstructor
+@ToString
+public class SyncEngineStatusDescriptor {
+
+ private int parallelTransferCount;
+ private int queueLenght;
+ private Map configuration;
+
+}
diff --git a/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties b/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties
index ccdd3c5..30064f1 100644
--- a/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties
+++ b/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties
@@ -1,6 +1,6 @@
scanner.pool.maxSize=10
-scanner.pool.coreSize=1
+scanner.pool.coreSize=10
scanner.pool.idle.ms=30000
transfers.pool.maxSize=40
-transfers.pool.coreSize=1
+transfers.pool.coreSize=20
transfers.pool.idle.ms=30000
\ No newline at end of file
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java b/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java
index 5121a67..a894a24 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java
@@ -1,40 +1,12 @@
package org.gcube.usecases.ws.thredds;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Collections;
-
-import org.apache.commons.io.IOUtils;
-import org.gcube.common.homelibrary.home.HomeLibrary;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Workspace;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
-import org.gcube.data.transfer.library.DataTransferClient;
-import org.gcube.data.transfer.library.faults.DestinationNotSetException;
-import org.gcube.data.transfer.library.faults.FailedTransferException;
-import org.gcube.data.transfer.library.faults.InitializationException;
-import org.gcube.data.transfer.library.faults.InvalidDestinationException;
-import org.gcube.data.transfer.library.faults.InvalidSourceException;
-import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
-import org.gcube.data.transfer.library.faults.SourceNotSetException;
-import org.gcube.data.transfer.library.faults.UnreachableNodeException;
-import org.gcube.data.transfer.model.Destination;
-import org.gcube.data.transfer.model.DestinationClashPolicy;
-import org.gcube.data.transfer.model.PluginInvocation;
-import org.gcube.usecases.ws.thredds.faults.InternalException;
-import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
+import org.gcube.common.storagehub.client.dsl.StorageHubClient;
+import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class DTTests {
- public static void main(String[] args) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException, MalformedURLException, FileNotFoundException, IOException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
+ public static void main(String[] args) throws Exception {
// TestCommons.setScope();
// String threddsHostname="thredds-d-d4s.d4science.org";
// DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+threddsHostname+":80");
@@ -57,8 +29,10 @@ public class DTTests {
//
TokenSetter.set("/gcube/devNext");
- Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
- String folderId=ws.getItemByPath("/Workspace/Accounting").getId();
+// Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
+ StorageHubClient client=WorkspaceUtils.getClient();
+
+ String folderId=TestCommons.getByPath("/Workspace/Accounting").getId();
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId);
System.out.println("Done");
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java b/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java
index 469c9db..c93885d 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java
@@ -1,44 +1,35 @@
package org.gcube.usecases.ws.thredds;
-import org.gcube.common.homelibrary.home.HomeLibrary;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Workspace;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.folder.FolderItem;
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FileContainer;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class GetWSSynchSpace {
- public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
- TokenSetter.set("/gcube/preprod/preVRE");
+ public static void main(String[] args) throws Exception {
+ TokenSetter.set("/gcube");
SyncEngine engine=SyncEngine.get();
- String folderId=TestCommons.getWSIdByPath("/Workspace/Thredds main catalog");
+ FolderContainer folder=(FolderContainer) TestCommons.getByPath("ThreddsDev");
- System.out.println(folderId);
-
-
- Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
- WorkspaceFolder folder=(WorkspaceFolder) ws.getItem(folderId);
+ System.out.println(folder.getId());
SynchFolderConfiguration config=new SynchFolderConfiguration();
System.out.println("Total size : "+computeLength(folder, config));
}
- private static final long computeLength(WorkspaceItem item,SynchFolderConfiguration config) throws InternalErrorException {
+ private static final long computeLength(ItemContainer> item,SynchFolderConfiguration config) throws StorageHubException{
long toReturn=0l;
- if(item.isFolder()) {
- for(WorkspaceItem child:((WorkspaceFolder)item).getChildren()) {
- if(item.isFolder()||config.matchesFilter(item.getName()))
+ if(item.getType().equals(ContainerType.FOLDER)) {
+ for(ItemContainer> child:((FolderContainer)item).list().withContent().getContainers()) {
+ if(child.getType().equals(ContainerType.FOLDER)||config.matchesFilter(child.get().getName()))
toReturn=toReturn+computeLength(child, config);
}
- }else toReturn=toReturn+((FolderItem)item).getLength();
+ }else toReturn=toReturn+((FileContainer)item).get().getContent().getSize();
return toReturn;
}
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java b/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java
index a3e57ee..cf98313 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java
@@ -4,23 +4,14 @@ import java.io.File;
import java.util.Map;
import java.util.Map.Entry;
-import org.gcube.common.homelibrary.home.HomeLibrary;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Properties;
-import org.gcube.common.homelibrary.home.workspace.Workspace;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FileContainer;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.client.dsl.StorageHubClient;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
+import org.gcube.common.storagehub.model.items.Item;
import org.gcube.data.transfer.library.DataTransferClient;
-import org.gcube.data.transfer.library.faults.DestinationNotSetException;
-import org.gcube.data.transfer.library.faults.FailedTransferException;
-import org.gcube.data.transfer.library.faults.InitializationException;
-import org.gcube.data.transfer.library.faults.InvalidDestinationException;
-import org.gcube.data.transfer.library.faults.InvalidSourceException;
-import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
@@ -28,16 +19,17 @@ import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
+import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class PublicLinkIssueTest {
- public static void main (String[] args) throws ItemNotFoundException, InternalErrorException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
+ public static void main (String[] args) throws Exception {
TokenSetter.set("/d4science.research-infrastructures.eu");
- Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
- System.out.println(ws.getItemByPath("/Workspace/ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc").getPublicLink(false));
+ StorageHubClient shClient=WorkspaceUtils.getClient();
+ System.out.println(((FileContainer)TestCommons.getByPath("ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc")).getPublicLink());
String threddsHostName = Commons.getThreddsHost();
@@ -51,7 +43,7 @@ public class PublicLinkIssueTest {
client.httpSource("http://data.d4science.org/V2drR2gxSFRTQlpLVC9nakozL29QcDdPR2U5UEVHYWRHbWJQNStIS0N6Yz0", dest);
- scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d"));
+ scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder());
System.out.println("OK FIRST ...");
try {
Metadata meta=SDIAbstractPlugin.metadata().build();
@@ -65,29 +57,29 @@ public class PublicLinkIssueTest {
e.printStackTrace();
}
System.out.println("CHECKING AGAIN");
- scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d"));
+ scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder());
}
- public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException {
- System.out.println("Folder "+folder.getPath());
- printProperties(folder.getProperties());
+ public static void scanForPrint(FolderContainer folder) throws StorageHubException {
+ System.out.println("Folder "+folder.get().getPath());
+ printProperties(folder.get());
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
- for(WorkspaceItem item:folder.getChildren())
- if(!item.isFolder()&&config.matchesFilter(item.getName())) {
+ for(ItemContainer> item:folder.list().withMetadata().getContainers())
+ if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) {
// System.out.println("ITEM "+item.getPath());
- printProperties(item.getProperties());
+ printProperties(item.get());
}
- for(WorkspaceItem item:folder.getChildren())
- if(item.isFolder())scanForPrint((WorkspaceFolder) item);
+ for(ItemContainer> item:folder.list().withMetadata().getContainers())
+ if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item);
}
- public static void printProperties(Properties prop) throws InternalErrorException {
- Map map=prop.getProperties();
+ public static void printProperties(Item toPrint){
+ Map map=toPrint.getMetadata().getMap();
// System.out.print("Properties : ..");
- for(Entry entry:map.entrySet()) {
+ for(Entry entry:map.entrySet()) {
if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java b/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java
index 871383e..3200c5f 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java
@@ -1,21 +1,13 @@
package org.gcube.usecases.ws.thredds;
-import java.io.IOException;
-import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Map;
-import org.gcube.common.homelibrary.home.HomeLibrary;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Workspace;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
+import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@@ -35,9 +27,9 @@ public class TestCommons {
String toCreateCatalogName;
- public String getFolderId() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
+ public String getFolderId() throws Exception {
if(folderId.startsWith("/"))
- return getWSIdByPath(folderId);
+ return getByPath(folderId).getId();
else return folderId;
}
@@ -45,26 +37,46 @@ public class TestCommons {
private static Map configs=new HashMap<>();
- private static String toUseConfig="pre";
+ private static String toUseConfig="default";
static {
- configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","",""));
+// configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","",""));
// folderName="WS-Tests";
- configs.put("default", new TestSet("Default Tests","/gcube/devsec/devVRE","/Workspace/ThreddsDev","public/netcdf","***REMOVED***","main"));
+ configs.put("simple", new TestSet("Simple label ","/gcube", "Test1","public/netcdf/simpleFolder","***REMOVED***","simple"));
+ configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
+ configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
- configs.put("pre", new TestSet("Default Tests","/gcube/preprod/preVRE","/Workspace/CMEMS","public/netcdf/CMEMS","***REMOVED***","main"));
+// configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main"));
+//
+//
+// configs.put("pre", new TestSet("Default Tests","/pred4s/preprod/preVRE","Thredds Synch","Fabio WS","91a8433b-9491-42c1-95e2-420245ef1943-980114272","SynchTestCatalog"));
}
- public static String getWSIdByPath(String path) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
- Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
- return ws.getItemByPath(path).getId();
+ public static ItemContainer> getByPath(String path) throws Exception {
+ StorageHubClient client = WorkspaceUtils.getClient();
+ FolderContainer root=client.getWSRoot();
+
+ return scan(root,path);
+ }
+
+
+ public static ItemContainer> scan(FolderContainer folder,String path) throws Exception{
+ String toLookFor=path.substring((path.startsWith("/")?1:0), path.length());
+ String[] split=toLookFor.split("/");
+ toLookFor=split[0];
+ for(ItemContainer> item:folder.list().withMetadata().getContainers())
+ if(item.get().getName().equals(toLookFor)) {
+ if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length()));
+ else return item;
+ }
+ throw new Exception("Unable to find "+path);
}
@@ -73,13 +85,13 @@ public class TestCommons {
}
- public static WorkspaceFolder getTestFolder() throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, IOException {
- Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
+ public static FolderContainer getTestFolder() throws Exception {
+
try{
- return (WorkspaceFolder) ws.getItem(configs.get(toUseConfig).getFolderId());
+ return WorkspaceUtils.getClient().open(configs.get(toUseConfig).getFolderId()).asFolder();
}catch(Throwable t) {
// try to use path
- return (WorkspaceFolder) ws.getItemByPath(configs.get(toUseConfig).getFolderId());
+ return (FolderContainer) getByPath(configs.get(toUseConfig).getFolderId());
}
@@ -101,12 +113,12 @@ public class TestCommons {
}
- public static ThreddsController getThreddsController() throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
+ public static ThreddsController getThreddsController() throws Exception {
SynchFolderConfiguration config=getSynchConfig();
return new ThreddsController(config.getRemotePath(), config.getTargetToken());
}
- public static SynchFolderConfiguration getSynchConfig() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
+ public static SynchFolderConfiguration getSynchConfig() throws Exception {
TestSet set=configs.get(toUseConfig);
return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId());
}
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java
index 9aed7fd..0d3fb5d 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java
@@ -1,18 +1,12 @@
package org.gcube.usecases.ws.thredds;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
-import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class ThreddsTests {
- public static void main(String[] args) throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
+ public static void main(String[] args) throws Exception{
TestCommons.setScope();
SynchFolderConfiguration folderConfig=TestCommons.getSynchConfig();
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java b/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java
index 8b1b0ed..f0b2971 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java
@@ -1,40 +1,31 @@
package org.gcube.usecases.ws.thredds;
-import java.io.IOException;
-import java.net.MalformedURLException;
-
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WSTimes {
- public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, InternalErrorException, HomeNotFoundException, UserNotFoundException, IOException {
+ public static void main(String[] args) throws Exception {
TestCommons.setScope();
- WorkspaceFolder folder=TestCommons.getTestFolder();
- for(WorkspaceItem item : folder.getChildren()) {
+ FolderContainer folder=TestCommons.getTestFolder();
+ for(ItemContainer> item : folder.list().withMetadata().getContainers()) {
printDates(item);
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
}
- item.getProperties().addProperty("Fake prop", "fake value");
+ WorkspaceUtils.addParameter(item,"Fake prop", "fake value");
printDates(item);
}
}
- public static final void printDates(WorkspaceItem item) throws InternalErrorException {
- System.out.println("ITEM : "+item.getName());
- System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getCreationTime().getTime()));
- System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getLastModificationTime().getTime()));
+ public static final void printDates(ItemContainer> item) {
+ System.out.println("ITEM : "+item.get().getName());
+ System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getCreationTime().getTime()));
+ System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getLastModificationTime().getTime()));
}
}
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java
index 77f96fc..e513098 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java
@@ -1,24 +1,14 @@
package org.gcube.usecases.ws.thredds;
-import java.io.IOException;
import java.util.Date;
-import org.gcube.common.homelibrary.home.HomeLibrary;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Workspace;
-import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WorkspaceAccounting {
- public static void main(String[] args) throws InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, IOException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, NumberFormatException, ItemNotFoundException {
+ public static void main(String[] args) throws Exception {
TestCommons.setScope();
- Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
+
// WorkspaceFolder folder=ws.getRoot().createFolder("Accounting", "test purposes");
@@ -68,7 +58,7 @@ public class WorkspaceAccounting {
- System.out.println(WorkspaceUtils.isModifiedAfter(ws.getItemByPath(TestCommons.getTestFolder().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc"), new Date(0l)));
+ System.out.println(WorkspaceUtils.isModifiedAfter(TestCommons.getByPath(TestCommons.getTestFolder().get().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc").get(), new Date(0l)));
}
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java
index b655557..599b6d1 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java
@@ -1,28 +1,24 @@
package org.gcube.usecases.ws.thredds;
-import java.io.IOException;
-import java.net.MalformedURLException;
-
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
-import org.gcube.usecases.ws.thredds.faults.InternalException;
-import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.model.Metadata;
public class WorkspaceCleanup {
- public static void main(String[] args) throws WorkspaceInteractionException, InternalException, InternalErrorException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, HomeNotFoundException, UserNotFoundException, IOException {
+ public static void main(String[] args) throws Exception{
TestCommons.setScope();
- WorkspaceFolder folder=TestCommons.getTestFolder();
+ FolderContainer folder=TestCommons.getTestFolder();
+
+ Metadata meta=folder.get().getMetadata();
+ meta.setMap(Constants.cleanedFolderPropertiesMap);
+ folder.setMetadata(meta);
SyncEngine engine=SyncEngine.get();
engine.forceUnlock(folder.getId());
- engine.unsetSynchronizedFolder(folder.getId(), false);
+// engine.unsetSynchronizedFolder(folder.getId(), false);
+
+//
+ WorkspaceProperties.printProperties(folder);
}
}
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java
index 2a92729..27e9002 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java
@@ -1,36 +1,23 @@
package org.gcube.usecases.ws.thredds;
-import java.io.IOException;
-import java.net.MalformedURLException;
import java.util.Collections;
-import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
-import org.gcube.usecases.ws.thredds.faults.InternalException;
-import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
-import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
-import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
+import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WorkspaceLock {
- public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, HomeNotFoundException, UserNotFoundException, WorkspaceInteractionException, InternalException, MalformedURLException, IOException {
+ public static void main(String[] args) throws Exception {
TestCommons.setScope();
- WorkspaceFolder folder=TestCommons.getTestFolder();
+ FolderContainer folder=TestCommons.getTestFolder();
WorkspaceFolderManager manager=new WorkspaceFolderManager(folder.getId());
String processID="mytest";
try {
System.out.println("Trying to cleanup, first.. ");
- folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null));
- System.out.println("FOLDER PROPERTIES : "+folder.getProperties().getProperties());
+ WorkspaceUtils.addParameters(folder, Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null));
+ System.out.println("FOLDER PROPERTIES : "+folder.get().getMetadata().getMap());
}catch(Throwable t) {
System.err.println("Dismiss error : ");
t.printStackTrace();
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java
index 737e094..7daf454 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java
@@ -1,35 +1,29 @@
package org.gcube.usecases.ws.thredds;
-import java.io.IOException;
-import java.net.MalformedURLException;
import java.util.Map;
import java.util.Map.Entry;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.Properties;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
-import org.gcube.usecases.ws.thredds.faults.InternalException;
-import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.model.exceptions.StorageHubException;
+import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceProperties {
- public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, ItemNotFoundException, MalformedURLException, IOException, WorkspaceInteractionException, InternalException {
+ public static void main(String[] args) throws Exception{
TestCommons.setScope();
- WorkspaceFolder folder=TestCommons.getTestFolder();
-
+ FolderContainer folder=TestCommons.getTestFolder();
+// System.out.println("Props : "+folder.get().getMetadata().getMap().size());
+// printProperties(folder);
+// System.out.println(new WorkspaceFolderManager(folder.getId()).isSynched());
SyncEngine.get().check(folder.getId(), true);
scanForPrint(folder);
+
// SyncEngine.get().shutDown();
// for(Workspace)
// printProperties(folder.getProperties());
@@ -61,24 +55,24 @@ public class WorkspaceProperties {
}
- public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException {
- System.out.println("Folder "+folder.getPath());
- printProperties(folder.getProperties());
+ public static void scanForPrint(FolderContainer folder) throws StorageHubException {
+ System.out.println("Folder "+folder.get().getPath());
+ printProperties(folder);
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
- for(WorkspaceItem item:folder.getChildren())
- if(!item.isFolder()&&config.matchesFilter(item.getName())) {
- System.out.println("ITEM "+item.getPath());
- printProperties(item.getProperties());
+ for(ItemContainer> item:folder.list().withMetadata().getContainers())
+ if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) {
+ System.out.println("ITEM "+item.get().getPath());
+ printProperties(item);
}
- for(WorkspaceItem item:folder.getChildren())
- if(item.isFolder())scanForPrint((WorkspaceFolder) item);
+ for(ItemContainer> item:folder.list().withMetadata().getContainers())
+ if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item);
}
- public static void printProperties(Properties prop) throws InternalErrorException {
- Map map=prop.getProperties();
+ public static void printProperties(ItemContainer> item) {
+ Map map=item.get().getMetadata().getMap();
System.out.print("Properties : ..");
- for(Entry entry:map.entrySet()) {
+ for(Entry entry:map.entrySet()) {
// if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {
diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java
index 2f23942..f9516ee 100644
--- a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java
+++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java
@@ -1,23 +1,10 @@
package org.gcube.usecases.ws.thredds;
-import java.io.IOException;
-import java.net.MalformedURLException;
import java.util.concurrent.Semaphore;
-import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
-import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
-import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
-import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
-import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
-import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
-import org.gcube.usecases.ws.thredds.faults.InternalException;
-import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
-import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
@@ -25,18 +12,20 @@ import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceSynchronizationTest {
- public static void main(String[] args) throws ProcessNotFoundException, InternalErrorException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, HomeNotFoundException, UserNotFoundException, MalformedURLException, IOException {
+ public static void main(String[] args) throws Exception {
// GET ENGINE : SINGLETON INSTANCE
- SyncEngine engine=SyncEngine.get();
+ final SyncEngine engine=SyncEngine.get();
engine.setRequestLogger("requests.txt");
//TEST INFO...
TestCommons.setScope();
- WorkspaceFolder folder=TestCommons.getTestFolder();
+ FolderContainer folder=TestCommons.getTestFolder();
// FOLDER CONFIGURATION BEAN
SynchFolderConfiguration config=TestCommons.getSynchConfig();
+// SynchFolderConfiguration config=null;
+
// try {
// //try to clean it up, first..
@@ -49,38 +38,40 @@ public class WorkspaceSynchronizationTest {
// engine.unsetSynchronizedFolder(folder.getId(), false);
// }
-
+ String folderId=folder.getId();
+// String folderId="15db7b0e-c215-41b7-b384-a1deda250b65";
try {
// WHEN OPENING A FOLDER, INVOKE CHECK TO UPDATE SYNCH STATUS
- engine.check(folder.getId(), false);
+ engine.check(folderId, false);
}catch(WorkspaceNotSynchedException e) {
System.out.println("Folder not synched, configurin it..");
- engine.setSynchronizedFolder(config, folder.getId());
- engine.check(folder.getId(), false);
+ engine.setSynchronizedFolder(config, folderId);
+ engine.check(folderId, false);
}catch(WorkspaceLockedException e) {
System.out.println("Workspace locked, going to force unlock.."); // MAINLY FOR TEST PURPOSES, OR WHEN SOMETHIGN GOES WRONG.. USE CAUTIOUSLY
- engine.forceUnlock(folder.getId());
- engine.check(folder.getId(), false);
+ engine.forceUnlock(folderId);
+ engine.check(folderId, false);
}
// INVOKE SYNCHRONIZATION ON FOLDER
- ProcessDescriptor descriptor=engine.doSync(folder.getId());
+ ProcessDescriptor descriptor=engine.doSync(folderId);
System.out.println("Obtained descriptor : "+descriptor);
Semaphore sem=new Semaphore(0);
// REGISTER CALLBACK TO MONITOR PROGRESS
- engine.registerCallBack(folder.getId(), new SyncOperationCallBack() {
+ engine.registerCallBack(folderId, new SyncOperationCallBack() {
@Override
public void onStep(ProcessStatus status, ProcessDescriptor descriptor) {
System.out.println("ON STEP : "+status+" "+descriptor);
+ System.out.println("ENGINE STATUS : "+engine.getStatus());
System.out.println("LOG : \n"+ status.getLogBuilder().toString());
if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release();
}
@@ -93,7 +84,7 @@ public class WorkspaceSynchronizationTest {
}
- engine.check(folder.getId(), true);
+ engine.check(folderId, true);
// INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES