gCubeSystem
/
ws-thredds
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merged 

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/spatial-data/ws-thredds@182034 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Fabio Sinibaldi 2019-10-15 10:08:09 +00:00
parent 98e50a055e
commit 80e201eab3
36 changed files with 962 additions and 865 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pom.xml
View File

@ -4,17 +4,17 @@
<parent> <parent>
<groupId>org.gcube.tools</groupId> <groupId>org.gcube.tools</groupId>
<artifactId>maven-parent</artifactId> <artifactId>maven-parent</artifactId>
<version>LATEST</version> <version>1.1.0</version>
</parent> </parent>
<groupId>org.gcube.spatial-data</groupId> <groupId>org.gcube.spatial.data</groupId>
<artifactId>ws-thredds</artifactId> <artifactId>ws-thredds</artifactId>
<version>0.2.0-SNAPSHOT</version> <version>0.2.2-SNAPSHOT</version>
<name>ws-thredds</name> <name>ws-thredds</name>
<description>prototype of WS integration with data-transfer for Thredds pubblication</description> <description>prototype of WS integration with data-transfer for Thredds pubblication</description>
<properties> <properties>
<distroDirectory>${project.basedir}/distro</distroDirectory> <distroDirectory>${project.basedir}/distro</distroDirectory>
<svnBaseUrl>http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId}</svnBaseUrl> <svnBaseUrl>http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId}</svnBaseUrl>
</properties> </properties>
@ -24,7 +24,17 @@
<url>${svnBaseUrl}/${project.artifactId}</url> <url>${svnBaseUrl}/${project.artifactId}</url>
</scm> </scm>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-portal-bom</artifactId>
<version>3.4.0-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies> <dependencies>
@ -32,38 +42,38 @@
<groupId>org.gcube.spatial.data</groupId> <groupId>org.gcube.spatial.data</groupId>
<artifactId>sdi-library</artifactId> <artifactId>sdi-library</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version> <version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.gcube.data.transfer</groupId> <groupId>org.gcube.data.transfer</groupId>
<artifactId>data-transfer-library</artifactId> <artifactId>data-transfer-library</artifactId>
<version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version> <version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency> </dependency>
<!-- <dependency> -->
<!-- <groupId>org.gcube.data.transfer</groupId> --> <!-- OLD HL -->
<!-- <artifactId>data-transfer-model</artifactId> --> <!-- <dependency> <groupId>org.gcube.common</groupId> <artifactId>home-library-jcr</artifactId>
<!-- <version>1.2.4-4.11.0-163203</version> --> <version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> </dependency> <dependency>
<!-- </dependency> --> <groupId>org.gcube.common</groupId> <artifactId>home-library</artifactId>
<dependency> <version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> </dependency> -->
<groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency> <dependency>
<groupId>org.gcube.common</groupId> <groupId>org.gcube.common</groupId>
<artifactId>home-library</artifactId> <artifactId>storagehub-client-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId> <artifactId>slf4j-log4j12</artifactId>
<version>1.6.4</version>
<scope>provided</scope>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>

0
requests.txt
View File

2
src/main/java/org/gcube/usecases/ws/thredds/Commons.java
View File

@ -14,7 +14,7 @@ import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget; import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import org.apache.tika.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.client.AuthorizationFilter; import org.gcube.data.transfer.library.client.AuthorizationFilter;

6
src/main/java/org/gcube/usecases/ws/thredds/Constants.java
View File

@ -55,11 +55,11 @@ public class Constants {
public static final String TRANSFERS_POOL_IDLE_MS="transfers.pool.idle.ms"; public static final String TRANSFERS_POOL_IDLE_MS="transfers.pool.idle.ms";
} }
public static final Map<String,String> cleanedItemPropertiesMap=new HashMap<String,String>(); public static final Map<String,Object> cleanedItemPropertiesMap=new HashMap<String,Object>();
public static final Map<String,String> cleanedFolderPropertiesMap=new HashMap<String,String>(); public static final Map<String,Object> cleanedFolderPropertiesMap=new HashMap<String,Object>();
public static final Map<String,String> defaultConfigurationMap=new HashMap<String,String>(); public static final Map<String,Object> defaultConfigurationMap=new HashMap<String,Object>();
static { static {
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.TBS, null); cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.TBS, null);

14
src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java
View File

@ -1,5 +1,7 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import lombok.Synchronized; import lombok.Synchronized;
@ -10,7 +12,7 @@ public class LocalConfiguration {
private static LocalConfiguration instance=null; private static LocalConfiguration instance=null;
@Synchronized @Synchronized
private static final LocalConfiguration get() { public static final LocalConfiguration get() {
if(instance==null) if(instance==null)
instance=new LocalConfiguration(); instance=new LocalConfiguration();
return instance; return instance;
@ -21,16 +23,22 @@ public class LocalConfiguration {
return (String) get().props.getOrDefault(property, Constants.defaultConfigurationMap.get(property)); return (String) get().props.getOrDefault(property, Constants.defaultConfigurationMap.get(property));
}catch(Throwable t) { }catch(Throwable t) {
log.warn("Unable to get configuration property "+property,t); log.warn("Unable to get configuration property "+property,t);
return Constants.defaultConfigurationMap.get(property); return Constants.defaultConfigurationMap.get(property)+"";
} }
} }
public Map<String,String> asMap(){
HashMap<String,String> toReturn=new HashMap<>();
for(Object key :props.keySet())
toReturn.put(key+"", (String)props.getOrDefault(key, Constants.defaultConfigurationMap.get(key)));
return toReturn;
}
//***************** INSTANCE //***************** INSTANCE
Properties props; Properties props;
public LocalConfiguration() { private LocalConfiguration() {
props=new Properties(); props=new Properties();
try{ try{
props.load(this.getClass().getResourceAsStream("configuration.properties")); props.load(this.getClass().getResourceAsStream("configuration.properties"));

11
src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java
View File

@ -38,20 +38,19 @@ public class NetUtils {
} }
public static String resolveRedirects(String url) throws IOException{ public static URL resolveRedirects(URL url) throws IOException{
log.debug("Resolving redirect for url {} ",url); log.debug("Resolving redirect for url {} ",url);
URL urlObj=new URL(url); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
int status=connection.getResponseCode(); int status=connection.getResponseCode();
if(status>=300&&status<400){ if(status>=300&&status<400){
String newUrl=connection.getHeaderField("Location"); String newUrl=connection.getHeaderField("Location");
log.debug("Following redirect from {} to {} ",url,newUrl); log.debug("Following redirect from {} to {} ",url,newUrl);
return resolveRedirects(newUrl); return resolveRedirects(new URL(newUrl));
}else return url; }else return url;
} }
public static File download(String url) throws MalformedURLException, IOException { public static File download(URL url) throws MalformedURLException, IOException {
return toFile(new URL(resolveRedirects(url)).openStream()); return toFile(resolveRedirects(url).openStream());
} }
} }

521
src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java
View File

@ -1,283 +1,258 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WrongItemTypeException;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
import org.gcube.usecases.ws.thredds.engine.PublishRequest;
import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
import org.gcube.usecases.ws.thredds.engine.PublishRequest.PublishItem;
import org.gcube.usecases.ws.thredds.engine.TransferRequestServer;
import org.gcube.usecases.ws.thredds.engine.TransferRequestServer.Report;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class PublishFolders { public class PublishFolders {
private static String WS_USER="fabio.sinibaldi"; // private static String WS_USER="fabio.sinibaldi";
//
//
public static void main(String[] args) { // public static void main(String[] args) {
TokenSetter.set("/d4science.research-infrastructures.eu"); // TokenSetter.set("/d4science.research-infrastructures.eu");
HashSet<FolderConfiguration> configs=new HashSet<>(); // HashSet<FolderConfiguration> configs=new HashSet<>();
//
//
// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS //// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS
// String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files // // String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files
String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST // String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST
//
// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET //// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET
//
String publishingUserToken="***REMOVED***"; //fabio @NextNext // String publishingUserToken="***REMOVED***"; //fabio @NextNext
//
//
FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE"); // FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE");
// folderConfig.setProvidedMetadata(true); // // folderConfig.setProvidedMetadata(true);
// folderConfig.setMetadataFolderId(metadataFolderID); // // folderConfig.setMetadataFolderId(metadataFolderID);
//
//
//
//
configs.add(folderConfig); // configs.add(folderConfig);
//
//
//
//
//
//
//
TransferRequestServer server=new TransferRequestServer(); // TransferRequestServer server=new TransferRequestServer();
for(FolderConfiguration entry:configs){ // for(FolderConfiguration entry:configs){
try{ // try{
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace(); // Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace();
// FolderReport report=new FolderReport(entry); // // FolderReport report=new FolderReport(entry);
log.info("Managing {} ",entry); // log.info("Managing {} ",entry);
WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId()); // WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId());
handleFolder(ws,entry,server,folder); // handleFolder(ws,entry,server,folder);
//
//
//
}catch(WorkspaceException e){ // }catch(WorkspaceException e){
System.err.println("WORKSPACE EXC "); // System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err); // e.printStackTrace(System.err);
}catch(HomeNotFoundException e){ // }catch(HomeNotFoundException e){
System.err.println("WORKSPACE EXC "); // System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err); // e.printStackTrace(System.err);
}catch(InternalErrorException e){ // }catch(InternalErrorException e){
System.err.println("WORKSPACE EXC "); // System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err); // e.printStackTrace(System.err);
}catch(UserNotFoundException e){ // }catch(UserNotFoundException e){
System.err.println("WORKSPACE EXC "); // System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err); // e.printStackTrace(System.err);
}catch(Exception e){ // }catch(Exception e){
System.err.println("UNEXPECTED EXC"); // System.err.println("UNEXPECTED EXC");
e.printStackTrace(System.err); // e.printStackTrace(System.err);
} // }
} // }
System.out.println("Waiting for service.. "); // System.out.println("Waiting for service.. ");
server.waitCompletion(); // server.waitCompletion();
//
Report report=server.getReport(); // Report report=server.getReport();
//
File reportFile =report.toFile(folderConfig); // File reportFile =report.toFile(folderConfig);
System.out.println("Report at "+reportFile.getAbsolutePath()); // System.out.println("Report at "+reportFile.getAbsolutePath());
//
//
} // }
//
/** // /**
* For *.nc | *.ncml // * For *.nc | *.ncml
* if relatedMetadataFolder contains <filename>.xml use meta // * if relatedMetadataFolder contains <filename>.xml use meta
* else ask DT to generate it // * else ask DT to generate it
* // *
* // *
* @param wsFolder // * @param wsFolder
* @param config // * @param config
* @param server // * @param server
* @return // * @return
* @throws WorkspaceException // * @throws WorkspaceException
* @throws InternalErrorException // * @throws InternalErrorException
* @throws ItemNotFoundException // * @throws ItemNotFoundException
* @throws HomeNotFoundException // * @throws HomeNotFoundException
* @throws UserNotFoundException // * @throws UserNotFoundException
*/ // */
//
public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ // public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
//
log.info("Handling folder {} ",folder.getPath()); // log.info("Handling folder {} ",folder.getPath());
//
cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName()); // cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName());
//
log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath()); // log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath());
publishFolder(ws,config,server,folder); // publishFolder(ws,config,server,folder);
} // }
//
//
public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ // public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
//
//
//
//
//Access folder // //Access folder
List<WorkspaceItem> folderItems=folder.getChildren(); // List<WorkspaceItem> folderItems=folder.getChildren();
//
if(config.isIncludeSubfolders()) { // if(config.isIncludeSubfolders()) {
log.info("Going through subfolders first....."); // log.info("Going through subfolders first.....");
for(WorkspaceItem item:folderItems) { // for(WorkspaceItem item:folderItems) {
try { // try {
if(item.isFolder()) { // if(item.isFolder()) {
FolderConfiguration subConfig=new FolderConfiguration(config); // FolderConfiguration subConfig=new FolderConfiguration(config);
subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName()); // subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName());
publishFolder(ws,subConfig,server,(WorkspaceFolder) item); // publishFolder(ws,subConfig,server,(WorkspaceFolder) item);
log.debug("Subfolder {} successfully analyzed ",item.getPath()); // log.debug("Subfolder {} successfully analyzed ",item.getPath());
} // }
}catch(Exception e) { // }catch(Exception e) {
try{ // try{
log.warn("Unabel to check item {} ",item.getPath(),e); // log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) { // }catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e); // log.warn("Unabel to check item and to get Path {} ",item,e);
} // }
} // }
} // }
} // }
//
//
//
log.debug("Checking for ncml files .... "); // log.debug("Checking for ncml files .... ");
for(WorkspaceItem item : folderItems) { // for(WorkspaceItem item : folderItems) {
try { // try {
if(!item.isFolder()) { // if(!item.isFolder()) {
String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); // String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
if(prefix.equals(".ncml")) { // if(prefix.equals(".ncml")) {
PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken()); // PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken());
if(config.isProvidedMetadata()) { // if(config.isProvidedMetadata()) {
String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; // String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); // File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
if (meta!=null) req.setMetadata(meta); // if (meta!=null) req.setMetadata(meta);
} // }
// TODO NB Check for queue // // TODO NB Check for queue
server.put(req); // server.put(req);
} // }
} // }
}catch(Exception e) { // }catch(Exception e) {
try{ // try{
log.warn("Unabel to check item {} ",item.getPath(),e); // log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) { // }catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e); // log.warn("Unabel to check item and to get Path {} ",item,e);
} // }
} // }
} // }
//
log.debug("Checking nc files.. "); // log.debug("Checking nc files.. ");
//
for(WorkspaceItem item:folder.getChildren()){ // for(WorkspaceItem item:folder.getChildren()){
try { // try {
if(!item.isFolder()) { // if(!item.isFolder()) {
String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); // String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
if(prefix.equals(".nc")){ // if(prefix.equals(".nc")){
// NC // // NC
PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken()); // PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken());
if(config.isProvidedMetadata()) { // if(config.isProvidedMetadata()) {
String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; // String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); // File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
if (meta!=null) req.setMetadata(meta); // if (meta!=null) req.setMetadata(meta);
} // }
server.put(req); // server.put(req);
} // }
} // }
}catch(Exception e) { // }catch(Exception e) {
try{ // try{
log.warn("Unabel to check item {} ",item.getPath(),e); // log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) { // }catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e); // log.warn("Unabel to check item and to get Path {} ",item,e);
} // }
} // }
} // }
folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+"")); // folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+""));
log.debug("Creating requests... "); // log.debug("Creating requests... ");
} // }
//
private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException { // private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException {
try{ // try{
WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId); // WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId);
if(found==null) throw new ItemNotFoundException("Found item was null"); // if(found==null) throw new ItemNotFoundException("Found item was null");
return NetUtils.toFile(((ExternalFile)found).getData()); // return NetUtils.toFile(((ExternalFile)found).getData());
}catch(ItemNotFoundException e) { // }catch(ItemNotFoundException e) {
return null; // return null;
} // }
} // }
//
// Remotely deletes folders which has history operations : RENAMING, REMOVAL // // Remotely deletes folders which has history operations : RENAMING, REMOVAL
private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException { // private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException {
List<AccountingEntry> history=folder.getAccounting(); // List<AccountingEntry> history=folder.getAccounting();
long lastUpdateTimeMillis=getLastUpdateTime(folder); // long lastUpdateTimeMillis=getLastUpdateTime(folder);
Date lastUpdate=new Date(lastUpdateTimeMillis); // Date lastUpdate=new Date(lastUpdateTimeMillis);
log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate)); // log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate));
//
//
//look into history // //look into history
boolean toDeleteCurrentFolder=false; // boolean toDeleteCurrentFolder=false;
for(AccountingEntry entry: history) { // for(AccountingEntry entry: history) {
Date eventTime=entry.getDate().getTime(); // Date eventTime=entry.getDate().getTime();
//
switch(entry.getEntryType()) { // switch(entry.getEntryType()) {
case REMOVAL: // case REMOVAL:
//
case RENAMING:{ // case RENAMING:{
log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); // log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
if(eventTime.after(lastUpdate)) { // if(eventTime.after(lastUpdate)) {
log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); // log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
toDeleteCurrentFolder=true; // toDeleteCurrentFolder=true;
} // }
} // }
} // }
if(toDeleteCurrentFolder) break; // if(toDeleteCurrentFolder) break;
} // }
//
//
//Delete Folder or scan children // //Delete Folder or scan children
if(toDeleteCurrentFolder) { // if(toDeleteCurrentFolder) {
log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath); // log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath);
try{ // try{
Commons.cleanupFolder(remoteFolderPath,targetToken); // Commons.cleanupFolder(remoteFolderPath,targetToken);
}catch(Throwable t) { // }catch(Throwable t) {
log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t); // log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t);
if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish // if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish
throw t; // throw t;
//
} // }
}else { // }else {
log.info("Folder is not to be cleaned up. Checking children.."); // log.info("Folder is not to be cleaned up. Checking children..");
for(WorkspaceItem item:folder.getChildren()) // for(WorkspaceItem item:folder.getChildren())
if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName()); // if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName());
} // }
} // }
//
private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException { // private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException {
try{ // try{
return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME)); // return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME));
}catch(Throwable e) { // }catch(Throwable e) {
log.debug("Unable to get last update time for {}. Considering 0..",item.getPath()); // log.debug("Unable to get last update time for {}. Considering 0..",item.getPath());
return 0l; // return 0l;
} // }
} // }
//
} }

3
src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java
View File

@ -9,6 +9,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.SynchEngineImpl;
import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException; import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor; import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -52,6 +53,6 @@ public interface SyncEngine {
public Set<CatalogBean> getAvailableCatalogsByToken(String token) throws InternalException; public Set<CatalogBean> getAvailableCatalogsByToken(String token) throws InternalException;
public SyncEngineStatusDescriptor getStatus();
} }

16
src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java
View File

@ -3,9 +3,6 @@ package org.gcube.usecases.ws.thredds.engine;
import java.io.File; import java.io.File;
import java.util.HashSet; import java.util.HashSet;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.Getter; import lombok.Getter;
@ -23,11 +20,14 @@ public class PublishRequest {
String name; String name;
String id; String id;
public PublishItem(WorkspaceItem item) throws InternalErrorException { // public PublishItem(Item item) throws InternalErrorException {
url=item.getPublicLink(false); //
name=item.getName(); //
id=item.getId(); //
} // url=item.getPublicLink(false);
// name=item.getName();
// id=item.getId();
// }
} }

15
src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java
View File

@ -1,17 +1,14 @@
package org.gcube.usecases.ws.thredds.engine; package org.gcube.usecases.ws.thredds.engine;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.faults.DestinationNotSetException; import org.gcube.data.transfer.library.faults.DestinationNotSetException;
@ -25,13 +22,7 @@ import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation; import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.spatial.data.sdi.utils.ScopeUtils; import org.gcube.spatial.data.sdi.utils.ScopeUtils;
import org.gcube.usecases.ws.thredds.Commons; import org.gcube.usecases.ws.thredds.Commons;
import org.gcube.usecases.ws.thredds.NetUtils; import org.gcube.usecases.ws.thredds.NetUtils;
@ -90,7 +81,7 @@ public class PublishThread implements Runnable {
waitFor(request.getQueueId(), request.getQueueCount()); waitFor(request.getQueueId(), request.getQueueCount());
log.debug("Loading netcdfFile .."); log.debug("Loading netcdfFile ..");
File ncmlFile=NetUtils.download(request.getSource().getUrl()); File ncmlFile=NetUtils.download(new URL(request.getSource().getUrl()));
String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath())); String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath()));
for(String reportId:request.getToGatherReportsId()) { for(String reportId:request.getToGatherReportsId()) {
@ -209,7 +200,7 @@ public class PublishThread implements Runnable {
log.debug("Alerting queue {}. Expected count is {} ",queueId,expected); log.debug("Alerting queue {}. Expected count is {} ",queueId,expected);
Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1)); Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1));
sem.release(); sem.release();
log.debug("Queue {} alerted. Remaining : {} out of {} ",queueId,sem.availablePermits(),expected); log.debug(String.format("Queue %1$s alerted. Remaining : %2$s out of %3$s ",queueId,sem.availablePermits(),expected));
} }

114
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java
View File

@ -11,9 +11,10 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status; import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status;
@ -22,6 +23,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest; import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferToThreddsRequest; import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferToThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
@ -37,7 +39,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class Process { public class Process {
private boolean submittedRequests=false; private boolean submittedRequests=false;
private ProcessDescriptor descriptor; private ProcessDescriptor descriptor;
@ -48,7 +50,7 @@ public class Process {
private Queue<StepReport> queuedReports=new LinkedList<>(); private Queue<StepReport> queuedReports=new LinkedList<>();
// private String folderId; // private String folderId;
private WorkspaceFolderManager manager; private WorkspaceFolderManager manager;
@ -58,13 +60,13 @@ public class Process {
public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException { public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException {
log.debug("Created Process with id {} ",processId); log.debug("Created Process with id {} ",processId);
// this.folderId=folderId; // this.folderId=folderId;
manager=new WorkspaceFolderManager(folderId); manager=new WorkspaceFolderManager(folderId);
manager.lock(processId); manager.lock(processId);
SynchFolderConfiguration folderConfig=manager.getSynchConfiguration(); SynchFolderConfiguration folderConfig=manager.getSynchConfiguration();
try { try {
descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().getPath(),System.currentTimeMillis(),processId,folderConfig); descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().get().getPath(),System.currentTimeMillis(),processId,folderConfig);
}catch(Exception e) { }catch(Exception e) {
throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e); throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e);
} }
@ -75,10 +77,10 @@ public class Process {
} }
public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalErrorException, InternalException { public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, StorageHubException, InternalException {
WorkspaceUtils.resetStatus(manager.getTheFolder()); WorkspaceUtils.resetStatus(manager.getTheFolder());
status.setCurrentMessage("Analyzing folder.."); status.setCurrentMessage("Analyzing folder..");
generateRequests(this,service, manager.getTheFolder()); generateRequests(this,service, manager.getTheFolder());
submittedRequests=true; submittedRequests=true;
if(status.getQueuedTransfers().get()>0) { if(status.getQueuedTransfers().get()>0) {
@ -164,6 +166,7 @@ public class Process {
break; break;
} }
} }
status.setCurrentMessage("Waiting for requests ["+(status.getErrorCount().get()+status.getServedTransfers().get())+"/"+status.getQueuedTransfers().get()+"] to be served.");
status.getLogBuilder().append( status.getLogBuilder().append(
String.format("%s - item [%s] %s: %s \n", Constants.DATE_FORMAT.format(new Date(report.getCompletionTime())), String.format("%s - item [%s] %s: %s \n", Constants.DATE_FORMAT.format(new Date(report.getCompletionTime())),
report.getElementName(),report.getStatus()+"",report.getMessage())); report.getElementName(),report.getStatus()+"",report.getMessage()));
@ -203,27 +206,27 @@ public class Process {
} }
private static final void generateRequests(Process ownerProcess,ExecutorService service,WorkspaceFolder toScanFolder ) throws InternalErrorException, InternalException{ private static final void generateRequests(Process ownerProcess,ExecutorService service,FolderContainer toScanFolder ) throws StorageHubException, InternalException{
String folderPath=toScanFolder.getPath(); String folderPath=toScanFolder.get().getPath();
log.info("Generating requests for folder {}",folderPath); log.info("Generating requests for folder {}",folderPath);
log.debug("Process is {} ",ownerProcess.getDescriptor()); log.debug("Process is {} ",ownerProcess.getDescriptor());
Set<String> handledWorkspaceItemEntries=new HashSet<String>(); Set<String> handledWorkspaceItemEntries=new HashSet<String>();
SynchFolderConfiguration config=ownerProcess.getDescriptor().getSynchConfiguration(); SynchFolderConfiguration config=ownerProcess.getDescriptor().getSynchConfiguration();
Set<String> remoteChildrenNames; Set<String> remoteChildrenNames;
Set<String> localChildrenNames=new HashSet<>(); Set<String> localChildrenNames=new HashSet<>();
List<WorkspaceItem> localFolderChildren=toScanFolder.getChildren(); List<ItemContainer<?>> localFolderChildren=toScanFolder.list().withAccounting().withMetadata().getContainers();
for(WorkspaceItem item:localFolderChildren) { for(ItemContainer<?> item:localFolderChildren) {
localChildrenNames.add(item.getName()); localChildrenNames.add(item.get().getName());
} }
String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); String relativePath=toScanFolder.get().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken()); ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken());
RemoteFileDescriptor folderDesc=null; RemoteFileDescriptor folderDesc=null;
try{ try{
folderDesc=folderController.getFileDescriptor(); folderDesc=folderController.getFileDescriptor();
@ -232,41 +235,41 @@ public class Process {
folderController.createEmptyFolder(null); folderController.createEmptyFolder(null);
folderDesc=folderController.getFileDescriptor(); folderDesc=folderController.getFileDescriptor();
} }
remoteChildrenNames=new HashSet<>(folderDesc.getChildren()); remoteChildrenNames=new HashSet<>(folderDesc.getChildren());
//*********************** HANDLING ACCOUNTING ENTRIES //*********************** HANDLING ACCOUNTING ENTRIES
Set<String> handledAccountingEntries=WorkspaceUtils.scanAccountingForStatus( toScanFolder, config, localChildrenNames, remoteChildrenNames, folderController, ownerProcess, service); Set<String> handledAccountingEntries=WorkspaceUtils.scanAccountingForStatus( toScanFolder, config, localChildrenNames, remoteChildrenNames, folderController, ownerProcess, service);
//SCAN FOLDER CONTENT //SCAN FOLDER CONTENT
log.debug("Checking content of {} ",folderPath); log.debug("Checking content of {} ",folderPath);
for(WorkspaceItem item:localFolderChildren) { for(ItemContainer<?> item:localFolderChildren) {
if(item.isFolder()) { if(item.getType().equals(ContainerType.FOLDER)) {
// RECURSIVE ON SUB FOLDERS // RECURSIVE ON SUB FOLDERS
generateRequests(ownerProcess,service,(WorkspaceFolder) item); generateRequests(ownerProcess,service,(FolderContainer) item);
}else { }else {
Map<String,String> props=item.getProperties().getProperties(); Map<String,Object> props=item.get().getMetadata().getMap();
String itemId=item.getId(); String itemId=item.getId();
String itemName=item.getName(); String itemName=item.get().getName();
// REQUESTS ARE EVALUATED ON PROPERTIES (SET BY PREVIOUS SCAN) // REQUESTS ARE EVALUATED ON PROPERTIES (SET BY PREVIOUS SCAN)
if(props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null)) { if(props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null)) {
try { try {
SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)); SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)+"");
log.trace("Found TBS item {}, name {}, status : ",item.getId(),item.getName(),status); log.trace(String.format("Found TBS item %1$s, name %2$s, status : %3$s",item.getId(),item.get().getName(),status));
SynchronizationRequest request=null; SynchronizationRequest request=null;
switch(status) { switch(status) {
case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder,item); case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder.get(),item.get());
break; break;
case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item, toScanFolder, null); case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item.get(), toScanFolder.get(), null);
break; break;
} }
@ -277,34 +280,45 @@ public class Process {
}else log.debug("Item is up to date"); }else log.debug("Item is up to date");
handledWorkspaceItemEntries.add(itemName); handledWorkspaceItemEntries.add(itemName);
}catch(Throwable t) { }catch(Throwable t) {
log.error("Unable to submit request for {} ID {} ",itemName,itemId,t); log.error(String.format("Unable to submit request for %1$s ID %2$s ",itemName,itemId),t);
} }
} }
} }
} }
// check items to be imported // check items to be imported
try { try {
Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, handledAccountingEntries, handledWorkspaceItemEntries, toScanFolder, folderController, config, ownerProcess, service); Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, handledAccountingEntries, handledWorkspaceItemEntries, toScanFolder, folderController, config, ownerProcess, service);
log.debug("Checking if remote location contains folders to be imported..."); log.debug("Checking if remote location contains folders to be imported...");
for(String item:toImportItems) { for(String item:toImportItems) {
if(folderController.getFileDescriptor(item).isDirectory()) { if(folderController.getFileDescriptor(item).isDirectory()) {
log.info("Creating folder {} under {} ",item,folderPath);
try{ //check if folder already existing!!
WorkspaceFolder folder=toScanFolder.createFolder(item, "Imported from thredds");
FolderContainer folder=null;
try{
try {
ItemContainer<?> foundElement=WorkspaceUtils.scan(toScanFolder, item);
if(!foundElement.getType().equals(ContainerType.FOLDER))
throw new Exception("Conflict on WS item "+foundElement.getId()+". Remote element is a folder ");
}catch(ItemNotFoundException e) {
log.info("Creating folder {} under {} ",item,folderPath);
folder=toScanFolder.newFolder(item, "Imported from thredds");
WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId()); WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId());
generateRequests(ownerProcess, service, folder); generateRequests(ownerProcess, service, folder);
}catch(Throwable t) {
log.error("Unable to import folder {} into {} ",item,folderPath);
} }
}catch(Throwable t) {
log.error("Unable to import folder {} into {} ",item,folderPath);
} }
} }
}
}catch(InternalException e) { }catch(InternalException e) {
log.error("Unable to check remote content with config {} ",config,e); log.error("Unable to check remote content with config {} ",config,e);
} }
log.info("All requests for {} synchronization have been submitted [count {} ]. ",folderPath,ownerProcess.status.getQueuedTransfers().get()); log.info("All requests for {} synchronization have been submitted [count {} ]. ",folderPath,ownerProcess.status.getQueuedTransfers().get());
} }

13
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java
View File

@ -7,7 +7,6 @@ import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -27,6 +26,7 @@ import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException; import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.CompletionCallback; import org.gcube.usecases.ws.thredds.model.CompletionCallback;
import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor; import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -53,15 +53,20 @@ public class SynchEngineImpl implements SyncEngine{
localProcesses=new ConcurrentHashMap<>(); localProcesses=new ConcurrentHashMap<>();
// NB UNBOUNDED QUEUE MEANS ONLY CORE THREADS ARE EXECUTED
int scannerMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_MAX_SIZE)); int scannerMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_MAX_SIZE));
int scannerCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_CORE_SIZE)); int scannerCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_CORE_SIZE));
// int scannerCoreSize=scannerMaxSize;
int scannerIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_IDLE_MS)); int scannerIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_IDLE_MS));
int transfersMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_MAX_SIZE)); int transfersMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_MAX_SIZE));
int transfersCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_CORE_SIZE)); int transfersCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_CORE_SIZE));
// int transfersCoreSize=transfersMaxSize;
int transfersIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_IDLE_MS)); int transfersIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_IDLE_MS));
initializationExecutor= new ThreadPoolExecutor(scannerCoreSize, scannerMaxSize, scannerIdleMs, initializationExecutor= new ThreadPoolExecutor(scannerCoreSize, scannerMaxSize, scannerIdleMs,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
@ -264,4 +269,10 @@ public class SynchEngineImpl implements SyncEngine{
return toReturn; return toReturn;
} }
@Override
public SyncEngineStatusDescriptor getStatus() {
ThreadPoolExecutor exec=(ThreadPoolExecutor) synchronizationExecutor;
return new SyncEngineStatusDescriptor(exec.getActiveCount(), exec.getQueue().size(), LocalConfiguration.get().asMap());
}
} }

11
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java
View File

@ -7,6 +7,7 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.util.Set; import java.util.Set;
@ -16,7 +17,8 @@ import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import org.apache.tika.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.library.TransferResult;
@ -83,10 +85,10 @@ public class ThreddsController {
private void resetCallerToken() { private void resetCallerToken() {
if(callerToken!=null) { if(callerToken!=null) {
log.trace("Resetting caller token {}. Target Token is {}, current is {} ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())); log.trace(String.format("Resetting caller token %1$s. Target Token is %2$s, current is %3$s ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())));
TokenSetter.setToken(callerToken); TokenSetter.setToken(callerToken);
callerToken=null; callerToken=null;
}else log.trace("Caller token {} already reset [current token {}]. Target Token is {}",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)); }else log.trace(String.format("Caller token %1$s already reset [current token %2$s]. Target Token is %3$s",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)));
} }
public final ThreddsInfo getThreddsInfo() { public final ThreddsInfo getThreddsInfo() {
@ -208,6 +210,7 @@ public class ThreddsController {
public ThreddsCatalog createCatalog(String name) throws InternalException { public ThreddsCatalog createCatalog(String name) throws InternalException {
setTargetToken(); setTargetToken();
try{ try{
SecurityTokenProvider.instance.get();
log.info("Creating catalog with name {} for path {} ",name,operatingPath); log.info("Creating catalog with name {} for path {} ",name,operatingPath);
String sdiUrl="http://"+getSDIServiceHost()+"/"+Constants.SDI_THREDDS_BASE_URL; String sdiUrl="http://"+getSDIServiceHost()+"/"+Constants.SDI_THREDDS_BASE_URL;
Response resp=getWebClient().target(sdiUrl). Response resp=getWebClient().target(sdiUrl).
@ -267,7 +270,7 @@ public class ThreddsController {
} }
public TransferResult transferFile(Destination dest,String url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { public TransferResult transferFile(Destination dest,URL url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
setTargetToken(); setTargetToken();
try{DataTransferClient client=getDTClient(hostname); try{DataTransferClient client=getDTClient(hostname);
if(invocations!=null&&!invocations.isEmpty()) if(invocations!=null&&!invocations.isEmpty())

119
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java
View File

@ -1,20 +1,17 @@
package org.gcube.usecases.ws.thredds.engine.impl; package org.gcube.usecases.ws.thredds.engine.impl;
import java.io.File; import java.io.File;
import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashSet; import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.homelibrary.home.workspace.Workspace; import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog; import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.Constants;
@ -43,7 +40,7 @@ public class WorkspaceFolderManager {
} }
private WorkspaceFolder theFolder; private FolderContainer theFolder;
private String folderId; private String folderId;
@ -52,20 +49,21 @@ public class WorkspaceFolderManager {
private SynchFolderConfiguration config=null; private SynchFolderConfiguration config=null;
private ThreddsController threddsController=null; private ThreddsController threddsController=null;
private Workspace ws; private StorageHubClient sc;
public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException { public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException {
try{ try{
ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); // ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
theFolder=(WorkspaceFolder) ws.getItem(folderId); sc=WorkspaceUtils.getClient();
theFolder=sc.open(folderId).asFolder();
this.folderId=folderId; this.folderId=folderId;
}catch(WorkspaceException | InternalErrorException | HomeNotFoundException | UserNotFoundException e) { }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to access folder id "+folderId,e); throw new WorkspaceInteractionException("Unable to access folder id "+folderId,e);
} }
} }
public WorkspaceFolder getTheFolder() { public FolderContainer getTheFolder() {
return theFolder; return theFolder;
} }
@ -79,21 +77,17 @@ public class WorkspaceFolderManager {
private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
try { try {
WorkspaceFolder root=(WorkspaceFolder) ws.getItem(getSynchConfiguration().getRootFolderId()); FolderContainer root=sc.open(getSynchConfiguration().getRootFolderId()).asFolder();
SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root); SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root);
return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken()); return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken());
}catch(WorkspaceException | InternalErrorException e) { }catch(StorageHubException e) {
throw new WorkspaceInteractionException(e); throw new WorkspaceInteractionException(e);
} }
} }
public boolean isRoot() throws WorkspaceNotSynchedException, WorkspaceInteractionException{ public boolean isRoot() throws WorkspaceNotSynchedException, WorkspaceInteractionException{
try{ return getSynchConfiguration().getRootFolderId().equals(theFolder.getId());
return getSynchConfiguration().getRootFolderId().equals(theFolder.getId());
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException(e);
}
} }
@ -103,20 +97,16 @@ public class WorkspaceFolderManager {
if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched."); if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched.");
log.debug("Loading properties for "); log.debug("Loading properties for ");
config=WorkspaceUtils.loadConfiguration(theFolder); config=WorkspaceUtils.loadConfiguration(theFolder);
}catch(InternalErrorException e) { }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e); throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e);
} }
} }
return config; return config;
} }
public boolean isSynched() throws WorkspaceInteractionException { public boolean isSynched(){
try{ Map<String,Object> props=theFolder.get().getMetadata().getMap();
Map<String,String> props=theFolder.getProperties().getProperties();
return props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null); return props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null);
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException("Unable to check Synch flag on "+folderId,e);
}
} }
@ -126,9 +116,9 @@ public class WorkspaceFolderManager {
SynchFolderConfiguration config=getSynchConfiguration(); SynchFolderConfiguration config=getSynchConfiguration();
try{ try{
checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder)); checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder.get()));
return new SyncFolderDescriptor(this.folderId,this.theFolder.getPath(),config); return new SyncFolderDescriptor(this.folderId,this.theFolder.get().getPath(),config);
}catch(InternalErrorException e) { }catch(StorageHubException e) {
throw new WorkspaceInteractionException(e); throw new WorkspaceInteractionException(e);
} }
} }
@ -188,7 +178,7 @@ public class WorkspaceFolderManager {
}catch(InternalException e) { }catch(InternalException e) {
throw new InternalException ("Unable to check/initialize remote folder",e); throw new InternalException ("Unable to check/initialize remote folder",e);
}catch(InternalErrorException e) { }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to set Properties to "+folderId,e); throw new WorkspaceInteractionException("Unable to set Properties to "+folderId,e);
} }
} }
@ -202,12 +192,12 @@ public class WorkspaceFolderManager {
WorkspaceUtils.cleanItem(theFolder); WorkspaceUtils.cleanItem(theFolder);
if(deleteRemote) if(deleteRemote)
getThreddsController().createEmptyFolder(null); getThreddsController().createEmptyFolder(null);
}catch(InternalErrorException e) { }catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to cleanup "+folderId,e); throw new WorkspaceInteractionException("Unable to cleanup "+folderId,e);
} }
} }
public void setLastUpdateTime() throws InternalErrorException { public void setLastUpdateTime() throws StorageHubException {
WorkspaceUtils.setLastUpdateTime(theFolder, System.currentTimeMillis()); WorkspaceUtils.setLastUpdateTime(theFolder, System.currentTimeMillis());
} }
@ -251,12 +241,12 @@ public class WorkspaceFolderManager {
private static void checkFolder(WorkspaceFolder folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws InternalErrorException, InternalException { private static void checkFolder(FolderContainer folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws StorageHubException, InternalException {
// Check folder configuration // Check folder configuration
log.trace("Checking folder {} ",folder.getPath()); log.trace("Checking folder {} ",folder.get().getPath());
log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder); log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder);
String folderName=folder.getName(); String folderName=folder.get().getName();
String currentRemotePath=rootConfig.getRemotePath()+((relativePathFromRootFolder==null)?"":"/"+relativePathFromRootFolder); String currentRemotePath=rootConfig.getRemotePath()+((relativePathFromRootFolder==null)?"":"/"+relativePathFromRootFolder);
@ -270,19 +260,19 @@ public class WorkspaceFolderManager {
log.debug("Initializing properties for {} ",folderName); log.debug("Initializing properties for {} ",folderName);
//INIT PROPERTIES IF NOT PRESENT //INIT PROPERTIES IF NOT PRESENT
if(!WorkspaceUtils.isConfigured(folder)) if(!WorkspaceUtils.isConfigured(folder.get()))
WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId); WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
for(WorkspaceItem item:folder.getChildren()) { for(ItemContainer<?> item:folder.list().withAccounting().withMetadata().getContainers()) {
String itemName=item.getName(); String itemName=item.get().getName();
String itemRelativePath=(relativePathFromRootFolder==null)?itemName:relativePathFromRootFolder+"/"+itemName; String itemRelativePath=(relativePathFromRootFolder==null)?itemName:relativePathFromRootFolder+"/"+itemName;
String itemRemotePath=currentRemotePath+"/"+itemName; String itemRemotePath=currentRemotePath+"/"+itemName;
if(item.isFolder()) { if(item.getType().equals(ContainerType.FOLDER)) {
if(recursive) if(recursive)
checkFolder((WorkspaceFolder) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine); checkFolder((FolderContainer) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine);
else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId); else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
}else if(rootConfig.matchesFilter(itemName)) { }else if(rootConfig.matchesFilter(itemName)) {
if(!WorkspaceUtils.isConfigured(item)) if(!WorkspaceUtils.isConfigured(item.get()))
WorkspaceUtils.initProperties(item, null, null, null,null,null,null); WorkspaceUtils.initProperties(item, null, null, null,null,null,null);
} }
currentFolderExistingItem.add(itemName); currentFolderExistingItem.add(itemName);
@ -305,31 +295,50 @@ public class WorkspaceFolderManager {
currentFolderExistingItem, remoteFolderItems, currentFolderExistingItem, remoteFolderItems,
controller, null, null); controller, null, null);
if(accountingEntries.isEmpty()) { if(accountingEntries.isEmpty()) {
log.debug("No accounting entries found"); log.debug("No Accounting Entries to be managed..");
folderStatus=SynchronizationStatus.UP_TO_DATE; folderStatus=SynchronizationStatus.UP_TO_DATE;
} }
// CHECK WS ITEMS // CHECK WS ITEMS
for(WorkspaceItem item:folder.getChildren()) for(ItemContainer<?> item:folder.list().withAccounting().withMetadata().getContainers())
if(item.isFolder()||rootConfig.matchesFilter(item.getName())) { if(item.getType().equals(ContainerType.FOLDER)||rootConfig.matchesFilter(item.get().getName())) {
SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item, remoteFolderItems, controller,lastUpdatedRoutine); SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item.get(), remoteFolderItems, controller,lastUpdatedRoutine);
item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+""));
Metadata meta=item.get().getMetadata();
Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+"");
meta.setMap(map);
item.setMetadata(meta);
folderStatus=folderStatus.equals(SynchronizationStatus.UP_TO_DATE)?itemStatus:folderStatus; folderStatus=folderStatus.equals(SynchronizationStatus.UP_TO_DATE)?itemStatus:folderStatus;
} }
// CHECK REMOTE FOLDER // CHECK REMOTE FOLDER
if(folderStatus.equals(SynchronizationStatus.UP_TO_DATE)) { if(folderStatus.equals(SynchronizationStatus.UP_TO_DATE)) {
Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, accountingEntries, currentFolderExistingItem, folder, controller, rootConfig, null, null); Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, accountingEntries, currentFolderExistingItem, folder, controller, rootConfig, null, null);
if(!toImportItems.isEmpty()) folderStatus=SynchronizationStatus.OUTDATED_WS; if(!toImportItems.isEmpty()) folderStatus=SynchronizationStatus.OUTDATED_WS;
} }
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+"")); Metadata meta=folder.get().getMetadata();
Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+"");
meta.setMap(map);
folder.setMetadata(meta);
}else { }else {
// Remote Folder not existing, set everything to OUTDATED_REMOTE // Remote Folder not existing, set everything to OUTDATED_REMOTE
for(WorkspaceItem item:folder.getChildren()) for(ItemContainer<?> item:folder.list().withMetadata().getContainers()) {
item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"")); Metadata meta=item.get().getMetadata();
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"")); Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"");
meta.setMap(map);
item.setMetadata(meta);
}
Metadata meta=folder.get().getMetadata();
Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"");
meta.setMap(map);
folder.setMetadata(meta);
} }

262
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java
View File

@ -1,26 +1,31 @@
package org.gcube.usecases.ws.thredds.engine.impl; package org.gcube.usecases.ws.thredds.engine.impl;
import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashSet; import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibary.model.items.type.WorkspaceItemType;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.workspace.Properties; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRemoval; import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRenaming; import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.nodes.accounting.AccountEntry;
import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRemoval;
import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRenaming;
import org.gcube.common.storagehub.model.types.WorkspaceItemType;
import org.gcube.data.transfer.model.RemoteFileDescriptor; import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest; import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread; import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest; import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport; import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -33,10 +38,12 @@ public class WorkspaceUtils {
public static StorageHubClient getClient() throws StorageHubException{
return new StorageHubClient();
}
/** /**
* Checks current accounting info in order to infere synchronization status. * Checks current accounting info in order to infer synchronization status.
* OwnerProcess and service parameters can be null for check purposes. * OwnerProcess and service parameters can be null for check purposes.
* *
* @param folderPath * @param folderPath
@ -52,53 +59,62 @@ public class WorkspaceUtils {
* @throws InternalErrorException * @throws InternalErrorException
*/ */
static Set<String> scanAccountingForStatus( static Set<String> scanAccountingForStatus(
WorkspaceFolder toScanFolder, FolderContainer toScanFolder,
SynchFolderConfiguration config, SynchFolderConfiguration config,
Set<String> localChildrenNames, Set<String> localChildrenNames,
Set<String> remoteChildrenNames, Set<String> remoteChildrenNames,
ThreddsController folderController, ThreddsController folderController,
Process ownerProcess, Process ownerProcess,
ExecutorService service) throws InternalErrorException{ ExecutorService service) throws StorageHubException{
Set<String> handledAccountingEntries=new HashSet<>(); Set<String> handledAccountingEntries=new HashSet<>();
log.debug("Checking history of {} ",toScanFolder.getPath()); FolderItem toScanFolderItem=toScanFolder.get();
log.debug("Checking history of {} ",toScanFolderItem.getPath());
String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); String relativePath=toScanFolderItem.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
Date folderLastUpdateTime=null; Date folderLastUpdateTime=null;
try{ try{
folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolder); folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolderItem);
}catch(Throwable t) { }catch(Throwable t) {
log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolder.getName(),t); log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolderItem.getName(),t);
folderLastUpdateTime=new Date(0l); folderLastUpdateTime=new Date(0l);
} }
// scanning for deletions // scanning for deletions
log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolder.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime)); log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolderItem.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime));
for(AccountingEntry entry:toScanFolder.getAccounting()) { for(AccountEntry entry:toScanFolderItem.getAccounting().getEntries()) {
try { try {
Date eventTime=entry.getDate().getTime(); Date eventTime=entry.getDate().getTime();
if(folderLastUpdateTime==null|| eventTime.after(folderLastUpdateTime)) { // SKIP IF ENTRY OLDER THAN LAST UPDATE TIME if(folderLastUpdateTime==null|| eventTime.after(folderLastUpdateTime)) { // SKIP IF ENTRY OLDER THAN LAST UPDATE TIME
String toDeleteRemote=null; String toDeleteRemote=null;
switch(entry.getEntryType()) { switch(entry.getType()) {
case CUT: case CUT:
case REMOVAL:{ case REMOVAL:{
AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry; AccountFolderEntryRemoval removalEntry=(AccountFolderEntryRemoval) entry;
if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER)|| if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER.toString())||
config.matchesFilter(removalEntry.getItemName())) config.matchesFilter(removalEntry.getItemName()))
toDeleteRemote=removalEntry.getItemName(); toDeleteRemote=removalEntry.getItemName();
break; break;
} }
case RENAMING:{ case RENAMING:{
AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry; AccountFolderEntryRenaming renamingEntry=(AccountFolderEntryRenaming) entry;
WorkspaceItem newItem=toScanFolder.find(renamingEntry.getNewItemName()); ItemContainer<?> newItem=toScanFolder.findByName(renamingEntry.getNewItemName()).getContainers().get(0);
if(newItem.isFolder()||config.matchesFilter(renamingEntry.getOldItemName())) if(newItem.getType().equals(ContainerType.FOLDER)||config.matchesFilter(renamingEntry.getOldItemName()))
toDeleteRemote=renamingEntry.getOldItemName(); toDeleteRemote=renamingEntry.getOldItemName();
break; break;
} }
default : {
log.debug("Skpping accounting entry {} ",entry.getType());
}
} }
@ -117,7 +133,7 @@ public class WorkspaceUtils {
if(service!=null) { if(service!=null) {
log.debug("Service is not null. Submitting request ... "); log.debug("Service is not null. Submitting request ... ");
if(eventTime.after(remoteDate)) { if(eventTime.after(remoteDate)) {
service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolder,toDeleteRemote))); service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolderItem,toDeleteRemote)));
handledAccountingEntries.add(toDeleteRemote); handledAccountingEntries.add(toDeleteRemote);
log.debug("Submitted DELETION request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet()); log.debug("Submitted DELETION request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
} }
@ -161,13 +177,15 @@ public class WorkspaceUtils {
RemoteFileDescriptor folderDesc, RemoteFileDescriptor folderDesc,
Set<String> handledAccountingEntries, Set<String> handledAccountingEntries,
Set<String> handledWorkspaceItemEntries, Set<String> handledWorkspaceItemEntries,
WorkspaceFolder toScanFolder, FolderContainer toScanFolder,
ThreddsController folderController, ThreddsController folderController,
SynchFolderConfiguration config, SynchFolderConfiguration config,
Process ownerProcess, Process ownerProcess,
ExecutorService service) throws InternalException, InternalErrorException{ ExecutorService service) throws InternalException, StorageHubException{
log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanFolder.getPath(),folderDesc.getAbsolutePath()); FolderItem toScanItem=toScanFolder.get();
log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanItem.getPath(),folderDesc.getAbsolutePath());
Set<String> handledRemoteElements=new HashSet<String>(); Set<String> handledRemoteElements=new HashSet<String>();
// String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); // String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
@ -189,7 +207,7 @@ public class WorkspaceUtils {
log.debug("Child {} matches filter..."); log.debug("Child {} matches filter...");
handledRemoteElements.add(child); handledRemoteElements.add(child);
if(service!=null) { if(service!=null) {
service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, child))); service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanItem, child)));
log.debug("Submitted IMPORT request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet()); log.debug("Submitted IMPORT request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
} }
// import if matching // import if matching
@ -201,10 +219,12 @@ public class WorkspaceUtils {
} }
static void initProperties(WorkspaceItem toInit, String remotePath, String filter, String targetToken, static void initProperties(ItemContainer<?> toInit, String remotePath, String filter, String targetToken,
String catalogName,Boolean validateMeta, String rootFolderId) throws InternalErrorException { String catalogName,Boolean validateMeta, String rootFolderId) throws StorageHubException {
Map<String,String> toSetProperties=toInit.getProperties().getProperties();
Metadata meta=toInit.get().getMetadata();
Map<String,Object> toSetProperties=meta.getMap();
initIfMissing(toSetProperties,Constants.WorkspaceProperties.TBS,"true"); initIfMissing(toSetProperties,Constants.WorkspaceProperties.TBS,"true");
@ -212,7 +232,7 @@ public class WorkspaceUtils {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_STATUS,StepReport.Status.OK+""); initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_STATUS,StepReport.Status.OK+"");
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.UP_TO_DATE+""); initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.UP_TO_DATE+"");
if(toInit.isFolder()) { if(toInit.getType().equals(ContainerType.FOLDER)) {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter); initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath); initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE); initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE);
@ -222,26 +242,28 @@ public class WorkspaceUtils {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId); initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId);
}else { }else {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.METADATA_UUID,null); initIfMissing(toSetProperties,Constants.WorkspaceProperties.METADATA_UUID,null);
} }
toInit.getProperties().addProperties(toSetProperties); meta.setMap(toSetProperties);
toInit.setMetadata(meta);
} }
private static void initIfMissing(Map<String,String> current,String key,String defaultValue) { private static void initIfMissing(Map<String,Object> current,String key,String defaultValue) {
if(!current.containsKey(key)|| if(!current.containsKey(key)||
current.get(key)==null|| current.get(key)==null||
current.get(key).equals("null")) current.put(key, defaultValue); current.get(key).equals("null")) current.put(key, defaultValue);
} }
static boolean isConfigured(WorkspaceItem toCheck) throws InternalErrorException { static boolean isConfigured(Item toCheck) throws StorageHubException {
return isConfigured(toCheck.getProperties().getProperties()); return isConfigured(toCheck.getMetadata().getMap());
} }
static boolean isConfigured(Map<String,String> toCheckProperties) { static boolean isConfigured(Map<String,Object> toCheckProperties) {
return (toCheckProperties.containsKey(Constants.WorkspaceProperties.TBS)&&toCheckProperties.get(Constants.WorkspaceProperties.TBS)!=null); return (toCheckProperties.containsKey(Constants.WorkspaceProperties.TBS)&&toCheckProperties.get(Constants.WorkspaceProperties.TBS)!=null);
} }
static SynchronizationStatus getStatusAgainstRemote(WorkspaceItem item, Set<String> existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException, InternalErrorException, RemoteFileNotFoundException { static SynchronizationStatus getStatusAgainstRemote(Item item, Set<String> existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException,RemoteFileNotFoundException {
String itemName=item.getName(); String itemName=item.getName();
SynchronizationStatus status=SynchronizationStatus.OUTDATED_REMOTE; SynchronizationStatus status=SynchronizationStatus.OUTDATED_REMOTE;
if(existingRemote.contains(itemName)) { if(existingRemote.contains(itemName)) {
@ -269,88 +291,132 @@ public class WorkspaceUtils {
} }
//
// /**
// *
// * @return max date between creation time, last modification time && LAST-UPDATE-PROP
// * @throws InternalErrorException
// * @throws NumberFormatException
// */
// static Date getMaxLastUpdate(WorkspaceItem item) throws NumberFormatException, InternalErrorException {
// return new Date(Long.max(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)),item.getLastModificationTime().getTimeInMillis()));
// }
static Date safelyGetLastUpdate(WorkspaceItem item) throws InternalErrorException { static Date safelyGetLastUpdate(Item item){
try { try {
return new Date(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME))); return new Date(Long.parseLong(item.getMetadata().getMap().get(Constants.WorkspaceProperties.LAST_UPDATE_TIME)+""));
}catch(NumberFormatException e) { }catch(NumberFormatException e) {
log.debug("Unable to get last update time for {} ",item.getName(),e); log.debug("Unable to get last update time for {} ",item.getName(),e);
return new Date(0l); return new Date(0l);
} }
} }
public static boolean isModifiedAfter(WorkspaceItem item,Date fromDate) throws InternalErrorException { public static boolean isModifiedAfter(Item item,Date fromDate) {
for(AccountingEntry entry:item.getAccounting()) { return item.getLastModificationTime().after(fromDate);
if(entry.getDate().getTime().after(fromDate)) { // for(AccountingEntry entry:item.getAccounting()) {
switch(entry.getEntryType()) { // if(entry.getDate().getTime().after(fromDate)) {
case PASTE: // switch(entry.getEntryType()) {
case CREATE: // case PASTE:
case RESTORE: // case CREATE:
case UPDATE: // case RESTORE:
case ADD: return true; // case UPDATE:
} // case ADD: return true;
} // }
} // }
return false; // }
// return false;
} }
static void cleanItem(WorkspaceItem item) throws InternalErrorException { static void cleanItem(ItemContainer<?> itemContainer) throws StorageHubException{
Properties props=item.getProperties(); Metadata meta=itemContainer.get().getMetadata();
if(props.hasProperty(Constants.WorkspaceProperties.TBS)) { Map<String,Object> map=meta.getMap();
if(item.isFolder()) { if(map.containsKey(Constants.WorkspaceProperties.TBS)) {
props.addProperties(Constants.cleanedFolderPropertiesMap); if(itemContainer.getType().equals(ContainerType.FOLDER)) {
for(WorkspaceItem child : ((WorkspaceFolder)item).getChildren()) map=Constants.cleanedFolderPropertiesMap;
for(ItemContainer<?> child : ((FolderContainer)itemContainer).list().withMetadata().getContainers())
cleanItem(child); cleanItem(child);
}else props.addProperties(Constants.cleanedItemPropertiesMap); }else map=Constants.cleanedItemPropertiesMap;
//Actually posting cleaned map
meta.setMap(map);
log.debug("Setting meta to item "+itemContainer.getId()+" : "+itemContainer.get().getPath());
itemContainer.setMetadata(meta);
} }
} }
static void setLastUpdateTime(WorkspaceFolder folder,long toSetTime) throws InternalErrorException { static void setLastUpdateTime(FolderContainer folder,long toSetTime) throws StorageHubException {
StepReport.Status currentWSStatus=StepReport.Status.valueOf(folder.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_STATUS));
Metadata meta=folder.get().getMetadata();
Map<String,Object> map=meta.getMap();
StepReport.Status currentWSStatus=StepReport.Status.valueOf(map.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+"");
if(currentWSStatus.equals(StepReport.Status.OK)) if(currentWSStatus.equals(StepReport.Status.OK))
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+"")); map.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+"");
for(WorkspaceItem item:folder.getChildren()) //Actually posting cleaned map
if(item.isFolder()) setLastUpdateTime((WorkspaceFolder) item, toSetTime); meta.setMap(map);
folder.setMetadata(meta);
// for(ItemContainer<?> child : folder.list().withMetadata().getContainers())
// if(child.getType().equals(ContainerType.FOLDER))
// setLastUpdateTime((FolderContainer) child, toSetTime);
} }
public static SynchFolderConfiguration loadConfiguration(WorkspaceItem item) throws InternalErrorException { public static SynchFolderConfiguration loadConfiguration(ItemContainer<?> item) throws StorageHubException {
if(item.isFolder()) { if(item.getType().equals(ContainerType.FOLDER)) {
Properties props=item.getProperties(); Map<String,Object> map=item.get().getMetadata().getMap();
SynchFolderConfiguration config=new SynchFolderConfiguration(); SynchFolderConfiguration config=new SynchFolderConfiguration();
config.setFilter(props.getPropertyValue(Constants.WorkspaceProperties.SYNCH_FILTER)); config.setFilter(""+map.get(Constants.WorkspaceProperties.SYNCH_FILTER));
config.setRemotePath(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH)); config.setRemotePath(""+map.get(Constants.WorkspaceProperties.REMOTE_PATH));
config.setRemotePersistence(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PERSISTENCE)); config.setRemotePersistence(""+map.get(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
config.setTargetToken(props.getPropertyValue(Constants.WorkspaceProperties.TARGET_TOKEN)); config.setTargetToken(""+map.get(Constants.WorkspaceProperties.TARGET_TOKEN));
config.setToCreateCatalogName(props.getPropertyValue(Constants.WorkspaceProperties.RELATED_CATALOG)); config.setToCreateCatalogName(""+map.get(Constants.WorkspaceProperties.RELATED_CATALOG));
config.setValidateMetadata(Boolean.parseBoolean(props.getPropertyValue(Constants.WorkspaceProperties.VALIDATE_METADATA))); config.setValidateMetadata(Boolean.parseBoolean(""+map.get(Constants.WorkspaceProperties.VALIDATE_METADATA)));
config.setRootFolderId(props.getPropertyValue(Constants.WorkspaceProperties.ROOT_FOLDER_ID)); config.setRootFolderId(""+map.get(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
return config; return config;
}else return loadConfiguration(item.getParent()); }else {
FolderContainer parentFolder=getClient().open(item.get().getParentId()).asFolder();
return loadConfiguration(parentFolder);
}
} }
static void resetStatus(WorkspaceItem item) throws InternalErrorException { static void resetStatus(ItemContainer<?> item) throws StorageHubException {
if(item.isFolder()) { if(item.getType().equals(ContainerType.FOLDER)) {
for(WorkspaceItem child: ((WorkspaceFolder)item).getChildren()) for(ItemContainer<?> child : ((FolderContainer)item).list().withMetadata().getContainers())
resetStatus(child); resetStatus(child);
} }
Map<String,String> props=item.getProperties().getProperties(); Metadata meta=item.get().getMetadata();
if(props.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) { Map<String,Object> map=meta.getMap();
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+""); if(map.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) {
item.getProperties().addProperties(props); map.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+"");
//Actually posting cleaned map
meta.setMap(map);
item.setMetadata(meta);
} }
} }
public static void addParameters(ItemContainer<?> item, Map<String,Object> toAdd) throws StorageHubException {
Metadata meta=item.get().getMetadata();
Map<String,Object> current=meta.getMap();
current.putAll(toAdd);
meta.setMap(current);
item.setMetadata(meta);
}
public static void addParameter(ItemContainer<?> item, String key, Object value) throws StorageHubException {
Metadata meta=item.get().getMetadata();
Map<String,Object> current=meta.getMap();
current.put(key,value);
meta.setMap(current);
item.setMetadata(meta);
}
public static ItemContainer<?> scan(FolderContainer folder,String path) throws ItemNotFoundException, StorageHubException{
String toLookFor=path.substring((path.startsWith("/")?1:0), path.length());
String[] split=toLookFor.split("/");
toLookFor=split[0];
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.get().getName().equals(toLookFor)) {
if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length()));
else return item;
}
throw new ItemNotFoundException("Unable to find "+path);
}
} }

4
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java
View File

@ -1,6 +1,6 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads; package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process; import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data; import lombok.Data;
@ -10,7 +10,7 @@ public class DeleteRemoteRequest extends SynchronizationRequest {
private String toRemoveName; private String toRemoveName;
public DeleteRemoteRequest(Process process,WorkspaceFolder location,String name) { public DeleteRemoteRequest(Process process,FolderItem location,String name) {
super(process,location); super(process,location);
this.toRemoveName=name; this.toRemoveName=name;
} }

4
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java
View File

@ -2,7 +2,7 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.usecases.ws.thredds.engine.impl.Process; import org.gcube.usecases.ws.thredds.engine.impl.Process;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
@ -45,7 +45,7 @@ public class ProcessInitializationThread implements Runnable {
} catch (InternalException e) { } catch (InternalException e) {
log.error("Unable to proceed..",e); log.error("Unable to proceed..",e);
theProcess.cancel(); theProcess.cancel();
} catch (InternalErrorException e) { } catch (StorageHubException e) {
log.error("Unable to proceed..",e); log.error("Unable to proceed..",e);
theProcess.cancel(); theProcess.cancel();
}catch(Throwable t) { }catch(Throwable t) {

31
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java
View File

@ -3,10 +3,12 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.FileWriter; import java.io.FileWriter;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.usecases.ws.thredds.Constants; import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.SyncEngine; import org.gcube.usecases.ws.thredds.SyncEngine;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import lombok.Synchronized; import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -81,33 +83,30 @@ public class RequestLogger {
} }
private static final String getRemotePath(WorkspaceItem item) { private static final String getRemotePath(Item item) {
try{ try{
if(item.isFolder()) return item.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); if(item instanceof FolderItem) return item.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
else return getRemotePath(item.getParent()); else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get());
}catch(InternalErrorException e) { }catch(StorageHubException e) {
log.warn("Unable to get Remote Path ",e); log.warn("Unable to get Remote Path ",e);
return "N/A"; return "N/A";
} }
} }
private static final String getName(WorkspaceItem item) { private static final String getName(Item item) {
try {
return item.getName(); return item.getName();
}catch(InternalErrorException e) {
log.warn("Unable to get name ",e);
return "N/A";
}
} }
private static final String getWSPath(WorkspaceItem item) { private static final String getWSPath(Item item) {
try{ try{
if(item.isFolder()) return item.getPath(); if(item instanceof FolderItem) return item.getPath();
else return getRemotePath(item.getParent()); else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get());
}catch(InternalErrorException e) { }catch(StorageHubException e) {
log.warn("Unable to get WS Path ",e); log.warn("Unable to get WS Path ",e);
return "N/A"; return "N/A";
} }
} }
} }

4
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java
View File

@ -1,6 +1,6 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads; package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process; import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data; import lombok.Data;
@ -12,5 +12,5 @@ public abstract class SynchronizationRequest {
@NonNull @NonNull
private Process process; private Process process;
@NonNull @NonNull
private WorkspaceFolder location; private FolderItem location;
} }

129
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java
View File

@ -8,9 +8,15 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.data.transfer.library.TransferResult; import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.data.transfer.model.DestinationClashPolicy;
@ -32,7 +38,6 @@ import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.CancellationException; import org.gcube.usecases.ws.thredds.faults.CancellationException;
import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError; import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport; import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.StepReport.OperationType; import org.gcube.usecases.ws.thredds.model.StepReport.OperationType;
@ -60,22 +65,25 @@ public class SynchronizationThread implements Runnable {
RequestLogger.get().log(theRequest); RequestLogger.get().log(theRequest);
String reportMessage="Never started"; String reportMessage="Never started";
String reportItemName="Still Unknown"; String reportItemName="Still Unknown";
Status toSetStatus=Status.ERROR; Status toSetStatus=Status.ERROR;
WorkspaceFolder parentFolder=theRequest.getLocation();
FolderItem parentFolderItem=theRequest.getLocation();
try { try {
StorageHubClient client=WorkspaceUtils.getClient();
FolderContainer parentFolder=client.open(parentFolderItem.getId()).asFolder();
checkCancelledProcess(); checkCancelledProcess();
SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(theRequest.getLocation()); SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(parentFolder);
ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken()); ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken());
if(theRequest instanceof TransferToThreddsRequest) { if(theRequest instanceof TransferToThreddsRequest) {
TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest; TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest;
WorkspaceItem item=request.getToTransfer(); Item item=request.getToTransfer();
FileContainer itemContainer=client.open(item.getId()).asFile();
//look for metadata in same folder //look for metadata in same folder
String itemName=item.getName(); String itemName=item.getName();
reportItemName=itemName; reportItemName=itemName;
String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml"; String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml";
WorkspaceItem metadataItem=getFileByName(item.getParent(),false,toLookMetadataName); FileContainer metadataItem=getFileByName(client.open(item.getParentId()).asFolder(),false,toLookMetadataName);
// if not present, generate with sis/geotk // if not present, generate with sis/geotk
Destination toSetDestination=new Destination(); Destination toSetDestination=new Destination();
@ -86,7 +94,7 @@ public class SynchronizationThread implements Runnable {
toSetDestination.setPersistenceId(synchConfig.getRemotePersistence()); toSetDestination.setPersistenceId(synchConfig.getRemotePersistence());
//NB ITEM IS SUPPOSED TO HAVE REMOTE PATH //NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
String fileLocation=request.getLocation().getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); String fileLocation=request.getLocation().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
toSetDestination.setSubFolder(fileLocation); toSetDestination.setSubFolder(fileLocation);
@ -108,17 +116,17 @@ public class SynchronizationThread implements Runnable {
checkCancelledProcess(); checkCancelledProcess();
TransferResult result=controller.transferFile(toSetDestination, item.getPublicLink(false), invocations); TransferResult result=controller.transferFile(toSetDestination, itemContainer.getPublicLink(), invocations);
Map<String,String> toSetProperties=new HashMap<String,String>(); Map<String,Object> toSetProperties=new HashMap<String,Object>();
String toSetMetadataUUID=null; String toSetMetadataUUID=null;
Boolean validateMetadata=synchConfig.getValidateMetadata(); Boolean validateMetadata=synchConfig.getValidateMetadata();
checkCancelledProcess(); checkCancelledProcess();
if(metadataItem==null) { if(metadataItem==null) {
ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID); ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID);
@ -134,15 +142,18 @@ public class SynchronizationThread implements Runnable {
toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID); toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID);
toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+""); toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+"");
item.getProperties().addProperties(toSetProperties); org.gcube.common.storagehub.model.Metadata meta=item.getMetadata();
meta.setMap(toSetProperties);
item.setMetadata(meta);
reportMessage="Successfully transferred and published."; reportMessage="Successfully transferred and published.";
// End ws->th // End ws->th
} else if(theRequest instanceof TransferFromThreddsRequest) { } else if(theRequest instanceof TransferFromThreddsRequest) {
Map<String,String> toSetProperties=new HashMap<String,String>(); Map<String,Object> toSetProperties=new HashMap<String,Object>();
TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest; TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest;
String toImportName=null; String toImportName=null;
WorkspaceItem targetItem=null; FileContainer targetItem=null;
if(importRequest.getTargetItem()==null) { if(importRequest.getTargetItem()==null) {
//Target Item will be created //Target Item will be created
toImportName=importRequest.getRemoteFilename(); toImportName=importRequest.getRemoteFilename();
@ -150,29 +161,38 @@ public class SynchronizationThread implements Runnable {
}else { }else {
//Target Item already exists //Target Item already exists
toImportName=importRequest.getTargetItem().getName(); toImportName=importRequest.getTargetItem().getName();
targetItem=importRequest.getTargetItem(); targetItem=client.open(importRequest.getTargetItem().getId()).asFile();
reportMessage="Updating file.."; reportMessage="Updating file..";
} }
reportItemName=toImportName; reportItemName=toImportName;
RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName); RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName);
toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+""); toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+"");
InputStream source=null; InputStream source=null;
try { try {
source=controller.getInputStream(toImportName); source=controller.getInputStream(toImportName);
if(targetItem==null) targetItem=parentFolder.uploadFile(source, toImportName, "Imported from Thredds");
targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source); // if(targetItem==null)
else // targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source);
targetItem.updateItem(source); // else {
targetItem.getProperties().addProperties(toSetProperties); // targetItem.updateItem(source);
// }
org.gcube.common.storagehub.model.Metadata meta=targetItem.get().getMetadata();
meta.setMap(toSetProperties);
targetItem.setMetadata(meta);
}finally { }finally {
if(source!=null) if(source!=null)
source.close(); source.close();
} }
reportMessage="File successfully imported"; reportMessage="File successfully imported";
}else if(theRequest instanceof DeleteRemoteRequest) { }else if(theRequest instanceof DeleteRemoteRequest) {
DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest; DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest;
reportItemName=deleteRequest.getToRemoveName(); reportItemName=deleteRequest.getToRemoveName();
@ -201,7 +221,7 @@ public class SynchronizationThread implements Runnable {
log.debug("Remote File not found ",e); log.debug("Remote File not found ",e);
reportMessage="Remote File not found : "+e.getMessage(); reportMessage="Remote File not found : "+e.getMessage();
toSetStatus=Status.ERROR; toSetStatus=Status.ERROR;
}catch(InternalErrorException e) { }catch(StorageHubException e) {
log.debug("Internal generic exception ",e); log.debug("Internal generic exception ",e);
reportMessage="Internal error : "+e.getMessage(); reportMessage="Internal error : "+e.getMessage();
toSetStatus=Status.ERROR; toSetStatus=Status.ERROR;
@ -210,7 +230,7 @@ public class SynchronizationThread implements Runnable {
reportMessage="Unexpected exception : "+t.getMessage(); reportMessage="Unexpected exception : "+t.getMessage();
toSetStatus=Status.ERROR; toSetStatus=Status.ERROR;
}finally { }finally {
updateParentProperty(parentFolder, toSetStatus); updateParentProperty(parentFolderItem, toSetStatus);
submitReport(reportItemName,reportMessage,toSetStatus); submitReport(reportItemName,reportMessage,toSetStatus);
ProcessIdProvider.instance.reset(); ProcessIdProvider.instance.reset();
} }
@ -218,19 +238,32 @@ public class SynchronizationThread implements Runnable {
@Synchronized @Synchronized
private static void updateParentProperty(WorkspaceFolder folder,StepReport.Status toSetStatus) { private static void updateParentProperty(FolderItem folderItem,StepReport.Status toSetStatus) {
try { boolean retry=false;
String currentValue=folder.getProperties().getProperties().get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS); do {
if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null")) try {
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"")); retry=false;
else { FolderContainer folder=WorkspaceUtils.getClient().open(folderItem.getId()).asFolder();
StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue); org.gcube.common.storagehub.model.Metadata folderMeta=folder.get().getMetadata();
if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus)) Map<String,Object> props=folderMeta.getMap();
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"")); String currentValue=props.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+"";
if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null"))
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
else {
StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue);
if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus))
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
}
folderMeta.setMap(props);
folder.setMetadata(folderMeta);
}catch(ItemLockedException e) {
log.debug("Item locked retrying.. ",e);
retry=true;
}catch(Throwable t) {
log.warn("Unable to update folder status ",t);
} }
}catch(Throwable t) { }while(retry);
log.warn("Unable to update folder status ",t);
}
} }
@ -254,11 +287,11 @@ public class SynchronizationThread implements Runnable {
private static MetadataReport publishMetadata(WorkspaceItem toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{ private static MetadataReport publishMetadata(FileContainer toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{
File tempMetaFile=null; File tempMetaFile=null;
try { try {
Metadata meta=SDIAbstractPlugin.metadata().build(); Metadata meta=SDIAbstractPlugin.metadata().build();
tempMetaFile=NetUtils.download(toPublish.getPublicLink(false)); tempMetaFile=NetUtils.download(toPublish.getPublicLink());
log.debug("Publishing metadata {} ",filename); log.debug("Publishing metadata {} ",filename);
@ -274,14 +307,14 @@ public class SynchronizationThread implements Runnable {
} }
private static final WorkspaceItem getFileByName(WorkspaceFolder toLookIntoFolder,boolean recursive,String toLookForName) throws InternalErrorException { private static final FileContainer getFileByName(FolderContainer toLookIntoFolder,boolean recursive,String toLookForName) throws StorageHubException {
log.debug("Looking for {} into {} [recursive {} ]",toLookForName,toLookIntoFolder.getPath()+" ID "+toLookIntoFolder.getId(),recursive); log.debug(String.format("Looking for %1$s into %2$s [recursive %3$s ]",toLookForName,toLookIntoFolder.get().getPath()+" ID "+toLookIntoFolder.getId(),recursive));
for(WorkspaceItem item : toLookIntoFolder.getChildren()) for(ItemContainer<?> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
if(!item.isFolder()&&item.getName().equals(toLookForName)) return item; if(!item.getType().equals(ContainerType.FOLDER)&&item.get().getName().equals(toLookForName)) return (FileContainer) item;
if(recursive) { if(recursive) {
for(WorkspaceItem item : toLookIntoFolder.getChildren()) for(ItemContainer<?> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
if(item.isFolder()) return getFileByName((WorkspaceFolder) item, recursive, toLookForName); if(item.getType().equals(ContainerType.FOLDER)) return getFileByName((FolderContainer) item, recursive, toLookForName);
} }
return null; return null;

17
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java
View File

@ -1,23 +1,32 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads; package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.storagehub.model.items.Item;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.Process; import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Data @Data
@Slf4j
public class TransferFromThreddsRequest extends SynchronizationRequest { public class TransferFromThreddsRequest extends SynchronizationRequest {
private WorkspaceItem targetItem; // can be null private Item targetItem; // can be null
private String remoteFilename; private String remoteFilename;
public TransferFromThreddsRequest(Process process, WorkspaceItem targetItem, WorkspaceFolder containingFolder, public TransferFromThreddsRequest(Process process, Item targetItem, FolderItem containingFolder,
String remoteFilename) { String remoteFilename) {
super(process,containingFolder); super(process,containingFolder);
this.targetItem = targetItem; this.targetItem = targetItem;
this.remoteFilename = remoteFilename; this.remoteFilename = remoteFilename;
if(targetItem==null)
log.debug(String.format("Created IMPORT request of %1$s into %1$s from %3$s",
remoteFilename,containingFolder.getPath(),containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
else
log.debug(String.format("Created UPDATE %1$s from %2$s", targetItem.getPath(),
containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
} }

14
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java
View File

@ -1,23 +1,27 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads; package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.storagehub.model.items.Item;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.Process; import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Getter @Getter
@Setter @Setter
@Slf4j
public class TransferToThreddsRequest extends SynchronizationRequest { public class TransferToThreddsRequest extends SynchronizationRequest {
private WorkspaceItem toTransfer; private Item toTransfer;
public TransferToThreddsRequest(Process process,WorkspaceFolder location, WorkspaceItem toTransfer) { public TransferToThreddsRequest(Process process,FolderItem location, Item toTransfer) {
super(process,location); super(process,location);
this.toTransfer = toTransfer; this.toTransfer = toTransfer;
log.debug(String.format("Created SEND request from $1%s to $2%s",
toTransfer.getPath(),location.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
} }
} }

35
src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java Normal file
View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class ItemNotFoundException extends WorkspaceInteractionException {
/**
*
*/
private static final long serialVersionUID = -6818076224344812668L;
public ItemNotFoundException() {
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

18
src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java Normal file
View File

@ -0,0 +1,18 @@
package org.gcube.usecases.ws.thredds.model;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.ToString;
@Getter
@AllArgsConstructor
@ToString
public class SyncEngineStatusDescriptor {
private int parallelTransferCount;
private int queueLenght;
private Map<String,String> configuration;
}

4
src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties
View File

@ -1,6 +1,6 @@
scanner.pool.maxSize=10 scanner.pool.maxSize=10
scanner.pool.coreSize=1 scanner.pool.coreSize=10
scanner.pool.idle.ms=30000 scanner.pool.idle.ms=30000
transfers.pool.maxSize=40 transfers.pool.maxSize=40
transfers.pool.coreSize=1 transfers.pool.coreSize=20
transfers.pool.idle.ms=30000 transfers.pool.idle.ms=30000

40
src/test/java/org/gcube/usecases/ws/thredds/DTTests.java
View File

@ -1,40 +1,12 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.File; import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import java.io.FileNotFoundException; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import org.apache.commons.io.IOUtils;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class DTTests { public class DTTests {
public static void main(String[] args) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException, MalformedURLException, FileNotFoundException, IOException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { public static void main(String[] args) throws Exception {
// TestCommons.setScope(); // TestCommons.setScope();
// String threddsHostname="thredds-d-d4s.d4science.org"; // String threddsHostname="thredds-d-d4s.d4science.org";
// DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+threddsHostname+":80"); // DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+threddsHostname+":80");
@ -57,8 +29,10 @@ public class DTTests {
// //
TokenSetter.set("/gcube/devNext"); TokenSetter.set("/gcube/devNext");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); // Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
String folderId=ws.getItemByPath("/Workspace/Accounting").getId(); StorageHubClient client=WorkspaceUtils.getClient();
String folderId=TestCommons.getByPath("/Workspace/Accounting").getId();
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId); SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId);
System.out.println("Done"); System.out.println("Done");

37
src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java
View File

@ -1,44 +1,35 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.homelibrary.home.workspace.Workspace; import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.homelibrary.home.workspace.folder.FolderItem;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class GetWSSynchSpace { public class GetWSSynchSpace {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { public static void main(String[] args) throws Exception {
TokenSetter.set("/gcube/preprod/preVRE"); TokenSetter.set("/gcube");
SyncEngine engine=SyncEngine.get(); SyncEngine engine=SyncEngine.get();
String folderId=TestCommons.getWSIdByPath("/Workspace/Thredds main catalog"); FolderContainer folder=(FolderContainer) TestCommons.getByPath("ThreddsDev");
System.out.println(folderId); System.out.println(folder.getId());
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
WorkspaceFolder folder=(WorkspaceFolder) ws.getItem(folderId);
SynchFolderConfiguration config=new SynchFolderConfiguration(); SynchFolderConfiguration config=new SynchFolderConfiguration();
System.out.println("Total size : "+computeLength(folder, config)); System.out.println("Total size : "+computeLength(folder, config));
} }
private static final long computeLength(WorkspaceItem item,SynchFolderConfiguration config) throws InternalErrorException { private static final long computeLength(ItemContainer<?> item,SynchFolderConfiguration config) throws StorageHubException{
long toReturn=0l; long toReturn=0l;
if(item.isFolder()) { if(item.getType().equals(ContainerType.FOLDER)) {
for(WorkspaceItem child:((WorkspaceFolder)item).getChildren()) { for(ItemContainer<?> child:((FolderContainer)item).list().withContent().getContainers()) {
if(item.isFolder()||config.matchesFilter(item.getName())) if(child.getType().equals(ContainerType.FOLDER)||config.matchesFilter(child.get().getName()))
toReturn=toReturn+computeLength(child, config); toReturn=toReturn+computeLength(child, config);
} }
}else toReturn=toReturn+((FolderItem)item).getLength(); }else toReturn=toReturn+((FileContainer)item).get().getContent().getSize();
return toReturn; return toReturn;
} }

56
src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java
View File

@ -4,23 +4,14 @@ import java.io.File;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.homelibrary.home.workspace.Properties; import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.homelibrary.home.workspace.Workspace; import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.library.DataTransferClient; import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.model.Destination; import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy; import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.spatial.data.sdi.interfaces.Metadata; import org.gcube.spatial.data.sdi.interfaces.Metadata;
@ -28,16 +19,17 @@ import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder; import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class PublicLinkIssueTest { public class PublicLinkIssueTest {
public static void main (String[] args) throws ItemNotFoundException, InternalErrorException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { public static void main (String[] args) throws Exception {
TokenSetter.set("/d4science.research-infrastructures.eu"); TokenSetter.set("/d4science.research-infrastructures.eu");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); StorageHubClient shClient=WorkspaceUtils.getClient();
System.out.println(ws.getItemByPath("/Workspace/ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc").getPublicLink(false)); System.out.println(((FileContainer)TestCommons.getByPath("ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc")).getPublicLink());
String threddsHostName = Commons.getThreddsHost(); String threddsHostName = Commons.getThreddsHost();
@ -51,7 +43,7 @@ public class PublicLinkIssueTest {
client.httpSource("http://data.d4science.org/V2drR2gxSFRTQlpLVC9nakozL29QcDdPR2U5UEVHYWRHbWJQNStIS0N6Yz0", dest); client.httpSource("http://data.d4science.org/V2drR2gxSFRTQlpLVC9nakozL29QcDdPR2U5UEVHYWRHbWJQNStIS0N6Yz0", dest);
scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d")); scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder());
System.out.println("OK FIRST ..."); System.out.println("OK FIRST ...");
try { try {
Metadata meta=SDIAbstractPlugin.metadata().build(); Metadata meta=SDIAbstractPlugin.metadata().build();
@ -65,29 +57,29 @@ public class PublicLinkIssueTest {
e.printStackTrace(); e.printStackTrace();
} }
System.out.println("CHECKING AGAIN"); System.out.println("CHECKING AGAIN");
scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d")); scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder());
} }
public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException { public static void scanForPrint(FolderContainer folder) throws StorageHubException {
System.out.println("Folder "+folder.getPath()); System.out.println("Folder "+folder.get().getPath());
printProperties(folder.getProperties()); printProperties(folder.get());
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "",""); SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
for(WorkspaceItem item:folder.getChildren()) for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(!item.isFolder()&&config.matchesFilter(item.getName())) { if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) {
// System.out.println("ITEM "+item.getPath()); // System.out.println("ITEM "+item.getPath());
printProperties(item.getProperties()); printProperties(item.get());
} }
for(WorkspaceItem item:folder.getChildren()) for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.isFolder())scanForPrint((WorkspaceFolder) item); if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item);
} }
public static void printProperties(Properties prop) throws InternalErrorException { public static void printProperties(Item toPrint){
Map<String,String> map=prop.getProperties(); Map<String,Object> map=toPrint.getMetadata().getMap();
// System.out.print("Properties : .."); // System.out.print("Properties : ..");
for(Entry<String,String> entry:map.entrySet()) { for(Entry<String,Object> entry:map.entrySet()) {
if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)|| if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)|| entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) { entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {

66
src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java
View File

@ -1,21 +1,13 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -35,9 +27,9 @@ public class TestCommons {
String toCreateCatalogName; String toCreateCatalogName;
public String getFolderId() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { public String getFolderId() throws Exception {
if(folderId.startsWith("/")) if(folderId.startsWith("/"))
return getWSIdByPath(folderId); return getByPath(folderId).getId();
else return folderId; else return folderId;
} }
@ -45,26 +37,46 @@ public class TestCommons {
private static Map<String,TestSet> configs=new HashMap<>(); private static Map<String,TestSet> configs=new HashMap<>();
private static String toUseConfig="pre"; private static String toUseConfig="default";
static { static {
configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","","")); // configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","",""));
// folderName="WS-Tests"; // folderName="WS-Tests";
configs.put("default", new TestSet("Default Tests","/gcube/devsec/devVRE","/Workspace/ThreddsDev","public/netcdf","***REMOVED***","main")); configs.put("simple", new TestSet("Simple label ","/gcube", "Test1","public/netcdf/simpleFolder","***REMOVED***","simple"));
configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
configs.put("pre", new TestSet("Default Tests","/gcube/preprod/preVRE","/Workspace/CMEMS","public/netcdf/CMEMS","***REMOVED***","main")); // configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main"));
//
//
// configs.put("pre", new TestSet("Default Tests","/pred4s/preprod/preVRE","Thredds Synch","Fabio WS","91a8433b-9491-42c1-95e2-420245ef1943-980114272","SynchTestCatalog"));
} }
public static String getWSIdByPath(String path) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { public static ItemContainer<?> getByPath(String path) throws Exception {
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); StorageHubClient client = WorkspaceUtils.getClient();
return ws.getItemByPath(path).getId(); FolderContainer root=client.getWSRoot();
return scan(root,path);
}
public static ItemContainer<?> scan(FolderContainer folder,String path) throws Exception{
String toLookFor=path.substring((path.startsWith("/")?1:0), path.length());
String[] split=toLookFor.split("/");
toLookFor=split[0];
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.get().getName().equals(toLookFor)) {
if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length()));
else return item;
}
throw new Exception("Unable to find "+path);
} }
@ -73,13 +85,13 @@ public class TestCommons {
} }
public static WorkspaceFolder getTestFolder() throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, IOException { public static FolderContainer getTestFolder() throws Exception {
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
try{ try{
return (WorkspaceFolder) ws.getItem(configs.get(toUseConfig).getFolderId()); return WorkspaceUtils.getClient().open(configs.get(toUseConfig).getFolderId()).asFolder();
}catch(Throwable t) { }catch(Throwable t) {
// try to use path // try to use path
return (WorkspaceFolder) ws.getItemByPath(configs.get(toUseConfig).getFolderId()); return (FolderContainer) getByPath(configs.get(toUseConfig).getFolderId());
} }
@ -101,12 +113,12 @@ public class TestCommons {
} }
public static ThreddsController getThreddsController() throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { public static ThreddsController getThreddsController() throws Exception {
SynchFolderConfiguration config=getSynchConfig(); SynchFolderConfiguration config=getSynchConfig();
return new ThreddsController(config.getRemotePath(), config.getTargetToken()); return new ThreddsController(config.getRemotePath(), config.getTargetToken());
} }
public static SynchFolderConfiguration getSynchConfig() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { public static SynchFolderConfiguration getSynchConfig() throws Exception {
TestSet set=configs.get(toUseConfig); TestSet set=configs.get(toUseConfig);
return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId()); return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId());
} }

8
src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java
View File

@ -1,18 +1,12 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class ThreddsTests { public class ThreddsTests {
public static void main(String[] args) throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { public static void main(String[] args) throws Exception{
TestCommons.setScope(); TestCommons.setScope();
SynchFolderConfiguration folderConfig=TestCommons.getSynchConfig(); SynchFolderConfiguration folderConfig=TestCommons.getSynchConfig();

31
src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java
View File

@ -1,40 +1,31 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.IOException; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import java.net.MalformedURLException; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
public class WSTimes { public class WSTimes {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, InternalErrorException, HomeNotFoundException, UserNotFoundException, IOException { public static void main(String[] args) throws Exception {
TestCommons.setScope(); TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder(); FolderContainer folder=TestCommons.getTestFolder();
for(WorkspaceItem item : folder.getChildren()) { for(ItemContainer<?> item : folder.list().withMetadata().getContainers()) {
printDates(item); printDates(item);
try { try {
Thread.sleep(3000); Thread.sleep(3000);
} catch (InterruptedException e) { } catch (InterruptedException e) {
} }
item.getProperties().addProperty("Fake prop", "fake value"); WorkspaceUtils.addParameter(item,"Fake prop", "fake value");
printDates(item); printDates(item);
} }
} }
public static final void printDates(WorkspaceItem item) throws InternalErrorException { public static final void printDates(ItemContainer<?> item) {
System.out.println("ITEM : "+item.getName()); System.out.println("ITEM : "+item.get().getName());
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getCreationTime().getTime())); System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getCreationTime().getTime()));
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getLastModificationTime().getTime())); System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getLastModificationTime().getTime()));
} }
} }

16
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java
View File

@ -1,24 +1,14 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.util.Date; import java.util.Date;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WorkspaceAccounting { public class WorkspaceAccounting {
public static void main(String[] args) throws InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, IOException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, NumberFormatException, ItemNotFoundException { public static void main(String[] args) throws Exception {
TestCommons.setScope(); TestCommons.setScope();
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
// WorkspaceFolder folder=ws.getRoot().createFolder("Accounting", "test purposes"); // WorkspaceFolder folder=ws.getRoot().createFolder("Accounting", "test purposes");
@ -68,7 +58,7 @@ public class WorkspaceAccounting {
System.out.println(WorkspaceUtils.isModifiedAfter(ws.getItemByPath(TestCommons.getTestFolder().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc"), new Date(0l))); System.out.println(WorkspaceUtils.isModifiedAfter(TestCommons.getByPath(TestCommons.getTestFolder().get().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc").get(), new Date(0l)));
} }

28
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java
View File

@ -1,28 +1,24 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.IOException; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import java.net.MalformedURLException; import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
public class WorkspaceCleanup { public class WorkspaceCleanup {
public static void main(String[] args) throws WorkspaceInteractionException, InternalException, InternalErrorException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, HomeNotFoundException, UserNotFoundException, IOException { public static void main(String[] args) throws Exception{
TestCommons.setScope(); TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder(); FolderContainer folder=TestCommons.getTestFolder();
Metadata meta=folder.get().getMetadata();
meta.setMap(Constants.cleanedFolderPropertiesMap);
folder.setMetadata(meta);
SyncEngine engine=SyncEngine.get(); SyncEngine engine=SyncEngine.get();
engine.forceUnlock(folder.getId()); engine.forceUnlock(folder.getId());
engine.unsetSynchronizedFolder(folder.getId(), false); // engine.unsetSynchronizedFolder(folder.getId(), false);
//
WorkspaceProperties.printProperties(folder);
} }
} }

25
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java
View File

@ -1,36 +1,23 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Collections; import java.util.Collections;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
import org.gcube.usecases.ws.thredds.faults.InternalException; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceLock { public class WorkspaceLock {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, HomeNotFoundException, UserNotFoundException, WorkspaceInteractionException, InternalException, MalformedURLException, IOException { public static void main(String[] args) throws Exception {
TestCommons.setScope(); TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder(); FolderContainer folder=TestCommons.getTestFolder();
WorkspaceFolderManager manager=new WorkspaceFolderManager(folder.getId()); WorkspaceFolderManager manager=new WorkspaceFolderManager(folder.getId());
String processID="mytest"; String processID="mytest";
try { try {
System.out.println("Trying to cleanup, first.. "); System.out.println("Trying to cleanup, first.. ");
folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null)); WorkspaceUtils.addParameters(folder, Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null));
System.out.println("FOLDER PROPERTIES : "+folder.getProperties().getProperties()); System.out.println("FOLDER PROPERTIES : "+folder.get().getMetadata().getMap());
}catch(Throwable t) { }catch(Throwable t) {
System.err.println("Dismiss error : "); System.err.println("Dismiss error : ");
t.printStackTrace(); t.printStackTrace();

52
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java
View File

@ -1,35 +1,29 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.homelibrary.home.workspace.Properties; import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceProperties { public class WorkspaceProperties {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, ItemNotFoundException, MalformedURLException, IOException, WorkspaceInteractionException, InternalException { public static void main(String[] args) throws Exception{
TestCommons.setScope(); TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder(); FolderContainer folder=TestCommons.getTestFolder();
// System.out.println("Props : "+folder.get().getMetadata().getMap().size());
// printProperties(folder);
// System.out.println(new WorkspaceFolderManager(folder.getId()).isSynched());
SyncEngine.get().check(folder.getId(), true); SyncEngine.get().check(folder.getId(), true);
scanForPrint(folder); scanForPrint(folder);
// SyncEngine.get().shutDown(); // SyncEngine.get().shutDown();
// for(Workspace) // for(Workspace)
// printProperties(folder.getProperties()); // printProperties(folder.getProperties());
@ -61,24 +55,24 @@ public class WorkspaceProperties {
} }
public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException { public static void scanForPrint(FolderContainer folder) throws StorageHubException {
System.out.println("Folder "+folder.getPath()); System.out.println("Folder "+folder.get().getPath());
printProperties(folder.getProperties()); printProperties(folder);
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "",""); SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
for(WorkspaceItem item:folder.getChildren()) for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(!item.isFolder()&&config.matchesFilter(item.getName())) { if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) {
System.out.println("ITEM "+item.getPath()); System.out.println("ITEM "+item.get().getPath());
printProperties(item.getProperties()); printProperties(item);
} }
for(WorkspaceItem item:folder.getChildren()) for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.isFolder())scanForPrint((WorkspaceFolder) item); if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item);
} }
public static void printProperties(Properties prop) throws InternalErrorException { public static void printProperties(ItemContainer<?> item) {
Map<String,String> map=prop.getProperties(); Map<String,Object> map=item.get().getMetadata().getMap();
System.out.print("Properties : .."); System.out.print("Properties : ..");
for(Entry<String,String> entry:map.entrySet()) { for(Entry<String,Object> entry:map.entrySet()) {
// if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)|| // if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)|| // entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) { // entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {

43
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java
View File

@ -1,23 +1,10 @@
package org.gcube.usecases.ws.thredds; package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus; import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException; import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
@ -25,18 +12,20 @@ import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceSynchronizationTest { public class WorkspaceSynchronizationTest {
public static void main(String[] args) throws ProcessNotFoundException, InternalErrorException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, HomeNotFoundException, UserNotFoundException, MalformedURLException, IOException { public static void main(String[] args) throws Exception {
// GET ENGINE : SINGLETON INSTANCE // GET ENGINE : SINGLETON INSTANCE
SyncEngine engine=SyncEngine.get(); final SyncEngine engine=SyncEngine.get();
engine.setRequestLogger("requests.txt"); engine.setRequestLogger("requests.txt");
//TEST INFO... //TEST INFO...
TestCommons.setScope(); TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder(); FolderContainer folder=TestCommons.getTestFolder();
// FOLDER CONFIGURATION BEAN // FOLDER CONFIGURATION BEAN
SynchFolderConfiguration config=TestCommons.getSynchConfig(); SynchFolderConfiguration config=TestCommons.getSynchConfig();
// SynchFolderConfiguration config=null;
// try { // try {
// //try to clean it up, first.. // //try to clean it up, first..
@ -49,38 +38,40 @@ public class WorkspaceSynchronizationTest {
// engine.unsetSynchronizedFolder(folder.getId(), false); // engine.unsetSynchronizedFolder(folder.getId(), false);
// } // }
String folderId=folder.getId();
// String folderId="15db7b0e-c215-41b7-b384-a1deda250b65";
try { try {
// WHEN OPENING A FOLDER, INVOKE CHECK TO UPDATE SYNCH STATUS // WHEN OPENING A FOLDER, INVOKE CHECK TO UPDATE SYNCH STATUS
engine.check(folder.getId(), false); engine.check(folderId, false);
}catch(WorkspaceNotSynchedException e) { }catch(WorkspaceNotSynchedException e) {
System.out.println("Folder not synched, configurin it.."); System.out.println("Folder not synched, configurin it..");
engine.setSynchronizedFolder(config, folder.getId()); engine.setSynchronizedFolder(config, folderId);
engine.check(folder.getId(), false); engine.check(folderId, false);
}catch(WorkspaceLockedException e) { }catch(WorkspaceLockedException e) {
System.out.println("Workspace locked, going to force unlock.."); // MAINLY FOR TEST PURPOSES, OR WHEN SOMETHIGN GOES WRONG.. USE CAUTIOUSLY System.out.println("Workspace locked, going to force unlock.."); // MAINLY FOR TEST PURPOSES, OR WHEN SOMETHIGN GOES WRONG.. USE CAUTIOUSLY
engine.forceUnlock(folder.getId()); engine.forceUnlock(folderId);
engine.check(folder.getId(), false); engine.check(folderId, false);
} }
// INVOKE SYNCHRONIZATION ON FOLDER // INVOKE SYNCHRONIZATION ON FOLDER
ProcessDescriptor descriptor=engine.doSync(folder.getId()); ProcessDescriptor descriptor=engine.doSync(folderId);
System.out.println("Obtained descriptor : "+descriptor); System.out.println("Obtained descriptor : "+descriptor);
Semaphore sem=new Semaphore(0); Semaphore sem=new Semaphore(0);
// REGISTER CALLBACK TO MONITOR PROGRESS // REGISTER CALLBACK TO MONITOR PROGRESS
engine.registerCallBack(folder.getId(), new SyncOperationCallBack() { engine.registerCallBack(folderId, new SyncOperationCallBack() {
@Override @Override
public void onStep(ProcessStatus status, ProcessDescriptor descriptor) { public void onStep(ProcessStatus status, ProcessDescriptor descriptor) {
System.out.println("ON STEP : "+status+" "+descriptor); System.out.println("ON STEP : "+status+" "+descriptor);
System.out.println("ENGINE STATUS : "+engine.getStatus());
System.out.println("LOG : \n"+ status.getLogBuilder().toString()); System.out.println("LOG : \n"+ status.getLogBuilder().toString());
if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release(); if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release();
} }
@ -93,7 +84,7 @@ public class WorkspaceSynchronizationTest {
} }
engine.check(folder.getId(), true); engine.check(folderId, true);
// INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES // INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES