gCubeSystem
/
ws-thredds
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merged 

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/spatial-data/ws-thredds@182034 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Fabio Sinibaldi 2019-10-15 10:08:09 +00:00
parent 98e50a055e
commit 80e201eab3
36 changed files with 962 additions and 865 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pom.xml
View File

@ -4,17 +4,17 @@
<parent>
<groupId>org.gcube.tools</groupId>
<artifactId>maven-parent</artifactId>
<version>LATEST</version>
<version>1.1.0</version>
</parent>
<groupId>org.gcube.spatial-data</groupId>
<groupId>org.gcube.spatial.data</groupId>
<artifactId>ws-thredds</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.2-SNAPSHOT</version>
<name>ws-thredds</name>
<description>prototype of WS integration with data-transfer for Thredds pubblication</description>
<properties>
<distroDirectory>${project.basedir}/distro</distroDirectory>
<svnBaseUrl>http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId}</svnBaseUrl>
<svnBaseUrl>http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId}</svnBaseUrl>
</properties>
@ -24,7 +24,17 @@
<url>${svnBaseUrl}/${project.artifactId}</url>
</scm>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-portal-bom</artifactId>
<version>3.4.0-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
@ -32,38 +42,38 @@
<groupId>org.gcube.spatial.data</groupId>
<artifactId>sdi-library</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>data-transfer-library</artifactId>
<version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<!-- <dependency> -->
<!-- <groupId>org.gcube.data.transfer</groupId> -->
<!-- <artifactId>data-transfer-model</artifactId> -->
<!-- <version>1.2.4-4.11.0-163203</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency>
<!-- OLD HL -->
<!-- <dependency> <groupId>org.gcube.common</groupId> <artifactId>home-library-jcr</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> </dependency> <dependency>
<groupId>org.gcube.common</groupId> <artifactId>home-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> </dependency> -->
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
<artifactId>storagehub-client-library</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.6.4</version>
<scope>provided</scope>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
</dependencies>
</project>

0
requests.txt
View File

2
src/main/java/org/gcube/usecases/ws/thredds/Commons.java
View File

@ -14,7 +14,7 @@ import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import org.apache.tika.io.IOUtils;
import org.apache.commons.io.IOUtils;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.client.AuthorizationFilter;

6
src/main/java/org/gcube/usecases/ws/thredds/Constants.java
View File

@ -55,11 +55,11 @@ public class Constants {
public static final String TRANSFERS_POOL_IDLE_MS="transfers.pool.idle.ms";
}
public static final Map<String,String> cleanedItemPropertiesMap=new HashMap<String,String>();
public static final Map<String,Object> cleanedItemPropertiesMap=new HashMap<String,Object>();
public static final Map<String,String> cleanedFolderPropertiesMap=new HashMap<String,String>();
public static final Map<String,Object> cleanedFolderPropertiesMap=new HashMap<String,Object>();
public static final Map<String,String> defaultConfigurationMap=new HashMap<String,String>();
public static final Map<String,Object> defaultConfigurationMap=new HashMap<String,Object>();
static {
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.TBS, null);

14
src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java
View File

@ -1,5 +1,7 @@
package org.gcube.usecases.ws.thredds;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import lombok.Synchronized;
@ -10,7 +12,7 @@ public class LocalConfiguration {
private static LocalConfiguration instance=null;
@Synchronized
private static final LocalConfiguration get() {
public static final LocalConfiguration get() {
if(instance==null)
instance=new LocalConfiguration();
return instance;
@ -21,16 +23,22 @@ public class LocalConfiguration {
return (String) get().props.getOrDefault(property, Constants.defaultConfigurationMap.get(property));
}catch(Throwable t) {
log.warn("Unable to get configuration property "+property,t);
return Constants.defaultConfigurationMap.get(property);
return Constants.defaultConfigurationMap.get(property)+"";
}
}
public Map<String,String> asMap(){
HashMap<String,String> toReturn=new HashMap<>();
for(Object key :props.keySet())
toReturn.put(key+"", (String)props.getOrDefault(key, Constants.defaultConfigurationMap.get(key)));
return toReturn;
}
//***************** INSTANCE
Properties props;
public LocalConfiguration() {
private LocalConfiguration() {
props=new Properties();
try{
props.load(this.getClass().getResourceAsStream("configuration.properties"));

11
src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java
View File

@ -38,20 +38,19 @@ public class NetUtils {
}
public static String resolveRedirects(String url) throws IOException{
public static URL resolveRedirects(URL url) throws IOException{
log.debug("Resolving redirect for url {} ",url);
URL urlObj=new URL(url);
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
int status=connection.getResponseCode();
if(status>=300&&status<400){
String newUrl=connection.getHeaderField("Location");
log.debug("Following redirect from {} to {} ",url,newUrl);
return resolveRedirects(newUrl);
return resolveRedirects(new URL(newUrl));
}else return url;
}
public static File download(String url) throws MalformedURLException, IOException {
return toFile(new URL(resolveRedirects(url)).openStream());
public static File download(URL url) throws MalformedURLException, IOException {
return toFile(resolveRedirects(url).openStream());
}
}

521
src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java
View File

@ -1,283 +1,258 @@
package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WrongItemTypeException;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
import org.gcube.usecases.ws.thredds.engine.PublishRequest;
import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
import org.gcube.usecases.ws.thredds.engine.PublishRequest.PublishItem;
import org.gcube.usecases.ws.thredds.engine.TransferRequestServer;
import org.gcube.usecases.ws.thredds.engine.TransferRequestServer.Report;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class PublishFolders {
private static String WS_USER="fabio.sinibaldi";
public static void main(String[] args) {
TokenSetter.set("/d4science.research-infrastructures.eu");
HashSet<FolderConfiguration> configs=new HashSet<>();
// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS
// String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files
String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST
// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET
String publishingUserToken="***REMOVED***"; //fabio @NextNext
FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE");
// folderConfig.setProvidedMetadata(true);
// folderConfig.setMetadataFolderId(metadataFolderID);
configs.add(folderConfig);
TransferRequestServer server=new TransferRequestServer();
for(FolderConfiguration entry:configs){
try{
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace();
// FolderReport report=new FolderReport(entry);
log.info("Managing {} ",entry);
WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId());
handleFolder(ws,entry,server,folder);
}catch(WorkspaceException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(HomeNotFoundException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(InternalErrorException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(UserNotFoundException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(Exception e){
System.err.println("UNEXPECTED EXC");
e.printStackTrace(System.err);
}
}
System.out.println("Waiting for service.. ");
server.waitCompletion();
Report report=server.getReport();
File reportFile =report.toFile(folderConfig);
System.out.println("Report at "+reportFile.getAbsolutePath());
}
/**
* For *.nc | *.ncml
* if relatedMetadataFolder contains <filename>.xml use meta
* else ask DT to generate it
*
*
* @param wsFolder
* @param config
* @param server
* @return
* @throws WorkspaceException
* @throws InternalErrorException
* @throws ItemNotFoundException
* @throws HomeNotFoundException
* @throws UserNotFoundException
*/
public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
log.info("Handling folder {} ",folder.getPath());
cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName());
log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath());
publishFolder(ws,config,server,folder);
}
public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
//Access folder
List<WorkspaceItem> folderItems=folder.getChildren();
if(config.isIncludeSubfolders()) {
log.info("Going through subfolders first.....");
for(WorkspaceItem item:folderItems) {
try {
if(item.isFolder()) {
FolderConfiguration subConfig=new FolderConfiguration(config);
subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName());
publishFolder(ws,subConfig,server,(WorkspaceFolder) item);
log.debug("Subfolder {} successfully analyzed ",item.getPath());
}
}catch(Exception e) {
try{
log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e);
}
}
}
}
log.debug("Checking for ncml files .... ");
for(WorkspaceItem item : folderItems) {
try {
if(!item.isFolder()) {
String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
if(prefix.equals(".ncml")) {
PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken());
if(config.isProvidedMetadata()) {
String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
if (meta!=null) req.setMetadata(meta);
}
// TODO NB Check for queue
server.put(req);
}
}
}catch(Exception e) {
try{
log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e);
}
}
}
log.debug("Checking nc files.. ");
for(WorkspaceItem item:folder.getChildren()){
try {
if(!item.isFolder()) {
String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
if(prefix.equals(".nc")){
// NC
PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken());
if(config.isProvidedMetadata()) {
String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
if (meta!=null) req.setMetadata(meta);
}
server.put(req);
}
}
}catch(Exception e) {
try{
log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e);
}
}
}
folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+""));
log.debug("Creating requests... ");
}
private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException {
try{
WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId);
if(found==null) throw new ItemNotFoundException("Found item was null");
return NetUtils.toFile(((ExternalFile)found).getData());
}catch(ItemNotFoundException e) {
return null;
}
}
// Remotely deletes folders which has history operations : RENAMING, REMOVAL
private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException {
List<AccountingEntry> history=folder.getAccounting();
long lastUpdateTimeMillis=getLastUpdateTime(folder);
Date lastUpdate=new Date(lastUpdateTimeMillis);
log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate));
//look into history
boolean toDeleteCurrentFolder=false;
for(AccountingEntry entry: history) {
Date eventTime=entry.getDate().getTime();
switch(entry.getEntryType()) {
case REMOVAL:
case RENAMING:{
log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
if(eventTime.after(lastUpdate)) {
log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
toDeleteCurrentFolder=true;
}
}
}
if(toDeleteCurrentFolder) break;
}
//Delete Folder or scan children
if(toDeleteCurrentFolder) {
log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath);
try{
Commons.cleanupFolder(remoteFolderPath,targetToken);
}catch(Throwable t) {
log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t);
if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish
throw t;
}
}else {
log.info("Folder is not to be cleaned up. Checking children..");
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName());
}
}
private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException {
try{
return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME));
}catch(Throwable e) {
log.debug("Unable to get last update time for {}. Considering 0..",item.getPath());
return 0l;
}
}
// private static String WS_USER="fabio.sinibaldi";
//
//
// public static void main(String[] args) {
// TokenSetter.set("/d4science.research-infrastructures.eu");
// HashSet<FolderConfiguration> configs=new HashSet<>();
//
//
//// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS
// // String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files
// String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST
//
//// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET
//
// String publishingUserToken="***REMOVED***"; //fabio @NextNext
//
//
// FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE");
// // folderConfig.setProvidedMetadata(true);
// // folderConfig.setMetadataFolderId(metadataFolderID);
//
//
//
//
// configs.add(folderConfig);
//
//
//
//
//
//
//
// TransferRequestServer server=new TransferRequestServer();
// for(FolderConfiguration entry:configs){
// try{
// Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace();
// // FolderReport report=new FolderReport(entry);
// log.info("Managing {} ",entry);
// WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId());
// handleFolder(ws,entry,server,folder);
//
//
//
// }catch(WorkspaceException e){
// System.err.println("WORKSPACE EXC ");
// e.printStackTrace(System.err);
// }catch(HomeNotFoundException e){
// System.err.println("WORKSPACE EXC ");
// e.printStackTrace(System.err);
// }catch(InternalErrorException e){
// System.err.println("WORKSPACE EXC ");
// e.printStackTrace(System.err);
// }catch(UserNotFoundException e){
// System.err.println("WORKSPACE EXC ");
// e.printStackTrace(System.err);
// }catch(Exception e){
// System.err.println("UNEXPECTED EXC");
// e.printStackTrace(System.err);
// }
// }
// System.out.println("Waiting for service.. ");
// server.waitCompletion();
//
// Report report=server.getReport();
//
// File reportFile =report.toFile(folderConfig);
// System.out.println("Report at "+reportFile.getAbsolutePath());
//
//
// }
//
// /**
// * For *.nc | *.ncml
// * if relatedMetadataFolder contains <filename>.xml use meta
// * else ask DT to generate it
// *
// *
// * @param wsFolder
// * @param config
// * @param server
// * @return
// * @throws WorkspaceException
// * @throws InternalErrorException
// * @throws ItemNotFoundException
// * @throws HomeNotFoundException
// * @throws UserNotFoundException
// */
//
// public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
//
// log.info("Handling folder {} ",folder.getPath());
//
// cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName());
//
// log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath());
// publishFolder(ws,config,server,folder);
// }
//
//
// public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
//
//
//
//
// //Access folder
// List<WorkspaceItem> folderItems=folder.getChildren();
//
// if(config.isIncludeSubfolders()) {
// log.info("Going through subfolders first.....");
// for(WorkspaceItem item:folderItems) {
// try {
// if(item.isFolder()) {
// FolderConfiguration subConfig=new FolderConfiguration(config);
// subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName());
// publishFolder(ws,subConfig,server,(WorkspaceFolder) item);
// log.debug("Subfolder {} successfully analyzed ",item.getPath());
// }
// }catch(Exception e) {
// try{
// log.warn("Unabel to check item {} ",item.getPath(),e);
// }catch(InternalErrorException e1) {
// log.warn("Unabel to check item and to get Path {} ",item,e);
// }
// }
// }
// }
//
//
//
// log.debug("Checking for ncml files .... ");
// for(WorkspaceItem item : folderItems) {
// try {
// if(!item.isFolder()) {
// String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
// if(prefix.equals(".ncml")) {
// PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken());
// if(config.isProvidedMetadata()) {
// String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
// File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
// if (meta!=null) req.setMetadata(meta);
// }
// // TODO NB Check for queue
// server.put(req);
// }
// }
// }catch(Exception e) {
// try{
// log.warn("Unabel to check item {} ",item.getPath(),e);
// }catch(InternalErrorException e1) {
// log.warn("Unabel to check item and to get Path {} ",item,e);
// }
// }
// }
//
// log.debug("Checking nc files.. ");
//
// for(WorkspaceItem item:folder.getChildren()){
// try {
// if(!item.isFolder()) {
// String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
// if(prefix.equals(".nc")){
// // NC
// PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken());
// if(config.isProvidedMetadata()) {
// String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
// File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
// if (meta!=null) req.setMetadata(meta);
// }
// server.put(req);
// }
// }
// }catch(Exception e) {
// try{
// log.warn("Unabel to check item {} ",item.getPath(),e);
// }catch(InternalErrorException e1) {
// log.warn("Unabel to check item and to get Path {} ",item,e);
// }
// }
// }
// folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+""));
// log.debug("Creating requests... ");
// }
//
// private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException {
// try{
// WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId);
// if(found==null) throw new ItemNotFoundException("Found item was null");
// return NetUtils.toFile(((ExternalFile)found).getData());
// }catch(ItemNotFoundException e) {
// return null;
// }
// }
//
// // Remotely deletes folders which has history operations : RENAMING, REMOVAL
// private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException {
// List<AccountingEntry> history=folder.getAccounting();
// long lastUpdateTimeMillis=getLastUpdateTime(folder);
// Date lastUpdate=new Date(lastUpdateTimeMillis);
// log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate));
//
//
// //look into history
// boolean toDeleteCurrentFolder=false;
// for(AccountingEntry entry: history) {
// Date eventTime=entry.getDate().getTime();
//
// switch(entry.getEntryType()) {
// case REMOVAL:
//
// case RENAMING:{
// log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
// if(eventTime.after(lastUpdate)) {
// log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
// toDeleteCurrentFolder=true;
// }
// }
// }
// if(toDeleteCurrentFolder) break;
// }
//
//
// //Delete Folder or scan children
// if(toDeleteCurrentFolder) {
// log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath);
// try{
// Commons.cleanupFolder(remoteFolderPath,targetToken);
// }catch(Throwable t) {
// log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t);
// if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish
// throw t;
//
// }
// }else {
// log.info("Folder is not to be cleaned up. Checking children..");
// for(WorkspaceItem item:folder.getChildren())
// if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName());
// }
// }
//
// private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException {
// try{
// return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME));
// }catch(Throwable e) {
// log.debug("Unable to get last update time for {}. Considering 0..",item.getPath());
// return 0l;
// }
// }
//
}

3
src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java
View File

@ -9,6 +9,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.SynchEngineImpl;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -52,6 +53,6 @@ public interface SyncEngine {
public Set<CatalogBean> getAvailableCatalogsByToken(String token) throws InternalException;
public SyncEngineStatusDescriptor getStatus();
}

16
src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java
View File

@ -3,9 +3,6 @@ package org.gcube.usecases.ws.thredds.engine;
import java.io.File;
import java.util.HashSet;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
@ -23,11 +20,14 @@ public class PublishRequest {
String name;
String id;
public PublishItem(WorkspaceItem item) throws InternalErrorException {
url=item.getPublicLink(false);
name=item.getName();
id=item.getId();
}
// public PublishItem(Item item) throws InternalErrorException {
//
//
//
// url=item.getPublicLink(false);
// name=item.getName();
// id=item.getId();
// }
}

15
src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java
View File

@ -1,17 +1,14 @@
package org.gcube.usecases.ws.thredds.engine;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.nio.file.Files;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
@ -25,13 +22,7 @@ import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
import org.gcube.usecases.ws.thredds.Commons;
import org.gcube.usecases.ws.thredds.NetUtils;
@ -90,7 +81,7 @@ public class PublishThread implements Runnable {
waitFor(request.getQueueId(), request.getQueueCount());
log.debug("Loading netcdfFile ..");
File ncmlFile=NetUtils.download(request.getSource().getUrl());
File ncmlFile=NetUtils.download(new URL(request.getSource().getUrl()));
String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath()));
for(String reportId:request.getToGatherReportsId()) {
@ -209,7 +200,7 @@ public class PublishThread implements Runnable {
log.debug("Alerting queue {}. Expected count is {} ",queueId,expected);
Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1));
sem.release();
log.debug("Queue {} alerted. Remaining : {} out of {} ",queueId,sem.availablePermits(),expected);
log.debug(String.format("Queue %1$s alerted. Remaining : %2$s out of %3$s ",queueId,sem.availablePermits(),expected));
}

114
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java
View File

@ -11,9 +11,10 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status;
@ -22,6 +23,7 @@ import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferToThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
@ -37,7 +39,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Process {
private boolean submittedRequests=false;
private ProcessDescriptor descriptor;
@ -48,7 +50,7 @@ public class Process {
private Queue<StepReport> queuedReports=new LinkedList<>();
// private String folderId;
// private String folderId;
private WorkspaceFolderManager manager;
@ -58,13 +60,13 @@ public class Process {
public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException {
log.debug("Created Process with id {} ",processId);
// this.folderId=folderId;
// this.folderId=folderId;
manager=new WorkspaceFolderManager(folderId);
manager.lock(processId);
SynchFolderConfiguration folderConfig=manager.getSynchConfiguration();
try {
descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().getPath(),System.currentTimeMillis(),processId,folderConfig);
descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().get().getPath(),System.currentTimeMillis(),processId,folderConfig);
}catch(Exception e) {
throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e);
}
@ -75,10 +77,10 @@ public class Process {
}
public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalErrorException, InternalException {
public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, StorageHubException, InternalException {
WorkspaceUtils.resetStatus(manager.getTheFolder());
status.setCurrentMessage("Analyzing folder..");
generateRequests(this,service, manager.getTheFolder());
submittedRequests=true;
if(status.getQueuedTransfers().get()>0) {
@ -164,6 +166,7 @@ public class Process {
break;
}
}
status.setCurrentMessage("Waiting for requests ["+(status.getErrorCount().get()+status.getServedTransfers().get())+"/"+status.getQueuedTransfers().get()+"] to be served.");
status.getLogBuilder().append(
String.format("%s - item [%s] %s: %s \n", Constants.DATE_FORMAT.format(new Date(report.getCompletionTime())),
report.getElementName(),report.getStatus()+"",report.getMessage()));
@ -203,27 +206,27 @@ public class Process {
}
private static final void generateRequests(Process ownerProcess,ExecutorService service,WorkspaceFolder toScanFolder ) throws InternalErrorException, InternalException{
String folderPath=toScanFolder.getPath();
private static final void generateRequests(Process ownerProcess,ExecutorService service,FolderContainer toScanFolder ) throws StorageHubException, InternalException{
String folderPath=toScanFolder.get().getPath();
log.info("Generating requests for folder {}",folderPath);
log.debug("Process is {} ",ownerProcess.getDescriptor());
Set<String> handledWorkspaceItemEntries=new HashSet<String>();
SynchFolderConfiguration config=ownerProcess.getDescriptor().getSynchConfiguration();
Set<String> remoteChildrenNames;
Set<String> localChildrenNames=new HashSet<>();
List<WorkspaceItem> localFolderChildren=toScanFolder.getChildren();
for(WorkspaceItem item:localFolderChildren) {
localChildrenNames.add(item.getName());
List<ItemContainer<?>> localFolderChildren=toScanFolder.list().withAccounting().withMetadata().getContainers();
for(ItemContainer<?> item:localFolderChildren) {
localChildrenNames.add(item.get().getName());
}
String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
String relativePath=toScanFolder.get().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken());
RemoteFileDescriptor folderDesc=null;
try{
folderDesc=folderController.getFileDescriptor();
@ -232,41 +235,41 @@ public class Process {
folderController.createEmptyFolder(null);
folderDesc=folderController.getFileDescriptor();
}
remoteChildrenNames=new HashSet<>(folderDesc.getChildren());
//*********************** HANDLING ACCOUNTING ENTRIES
Set<String> handledAccountingEntries=WorkspaceUtils.scanAccountingForStatus( toScanFolder, config, localChildrenNames, remoteChildrenNames, folderController, ownerProcess, service);
//SCAN FOLDER CONTENT
log.debug("Checking content of {} ",folderPath);
for(WorkspaceItem item:localFolderChildren) {
for(ItemContainer<?> item:localFolderChildren) {
if(item.isFolder()) {
if(item.getType().equals(ContainerType.FOLDER)) {
// RECURSIVE ON SUB FOLDERS
generateRequests(ownerProcess,service,(WorkspaceFolder) item);
generateRequests(ownerProcess,service,(FolderContainer) item);
}else {
Map<String,String> props=item.getProperties().getProperties();
Map<String,Object> props=item.get().getMetadata().getMap();
String itemId=item.getId();
String itemName=item.getName();
String itemName=item.get().getName();
// REQUESTS ARE EVALUATED ON PROPERTIES (SET BY PREVIOUS SCAN)
if(props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null)) {
try {
SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS));
log.trace("Found TBS item {}, name {}, status : ",item.getId(),item.getName(),status);
SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)+"");
log.trace(String.format("Found TBS item %1$s, name %2$s, status : %3$s",item.getId(),item.get().getName(),status));
SynchronizationRequest request=null;
switch(status) {
case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder,item);
case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder.get(),item.get());
break;
case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item, toScanFolder, null);
case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item.get(), toScanFolder.get(), null);
break;
}
@ -277,34 +280,45 @@ public class Process {
}else log.debug("Item is up to date");
handledWorkspaceItemEntries.add(itemName);
}catch(Throwable t) {
log.error("Unable to submit request for {} ID {} ",itemName,itemId,t);
log.error(String.format("Unable to submit request for %1$s ID %2$s ",itemName,itemId),t);
}
}
}
}
// check items to be imported
try {
Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, handledAccountingEntries, handledWorkspaceItemEntries, toScanFolder, folderController, config, ownerProcess, service);
log.debug("Checking if remote location contains folders to be imported...");
for(String item:toImportItems) {
if(folderController.getFileDescriptor(item).isDirectory()) {
log.info("Creating folder {} under {} ",item,folderPath);
try{
WorkspaceFolder folder=toScanFolder.createFolder(item, "Imported from thredds");
for(String item:toImportItems) {
if(folderController.getFileDescriptor(item).isDirectory()) {
//check if folder already existing!!
FolderContainer folder=null;
try{
try {
ItemContainer<?> foundElement=WorkspaceUtils.scan(toScanFolder, item);
if(!foundElement.getType().equals(ContainerType.FOLDER))
throw new Exception("Conflict on WS item "+foundElement.getId()+". Remote element is a folder ");
}catch(ItemNotFoundException e) {
log.info("Creating folder {} under {} ",item,folderPath);
folder=toScanFolder.newFolder(item, "Imported from thredds");
WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId());
generateRequests(ownerProcess, service, folder);
}catch(Throwable t) {
log.error("Unable to import folder {} into {} ",item,folderPath);
}
}catch(Throwable t) {
log.error("Unable to import folder {} into {} ",item,folderPath);
}
}
}
}catch(InternalException e) {
log.error("Unable to check remote content with config {} ",config,e);
}
log.info("All requests for {} synchronization have been submitted [count {} ]. ",folderPath,ownerProcess.status.getQueuedTransfers().get());
}

13
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java
View File

@ -7,7 +7,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ -27,6 +26,7 @@ import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.CompletionCallback;
import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -53,15 +53,20 @@ public class SynchEngineImpl implements SyncEngine{
localProcesses=new ConcurrentHashMap<>();
// NB UNBOUNDED QUEUE MEANS ONLY CORE THREADS ARE EXECUTED
int scannerMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_MAX_SIZE));
int scannerCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_CORE_SIZE));
// int scannerCoreSize=scannerMaxSize;
int scannerIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_IDLE_MS));
int transfersMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_MAX_SIZE));
int transfersCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_CORE_SIZE));
// int transfersCoreSize=transfersMaxSize;
int transfersIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_IDLE_MS));
initializationExecutor= new ThreadPoolExecutor(scannerCoreSize, scannerMaxSize, scannerIdleMs,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
@ -264,4 +269,10 @@ public class SynchEngineImpl implements SyncEngine{
return toReturn;
}
@Override
public SyncEngineStatusDescriptor getStatus() {
ThreadPoolExecutor exec=(ThreadPoolExecutor) synchronizationExecutor;
return new SyncEngineStatusDescriptor(exec.getActiveCount(), exec.getQueue().size(), LocalConfiguration.get().asMap());
}
}

11
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java
View File

@ -7,6 +7,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.net.URL;
import java.nio.file.Files;
import java.util.Set;
@ -16,7 +17,8 @@ import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.tika.io.IOUtils;
import org.apache.commons.io.IOUtils;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
@ -83,10 +85,10 @@ public class ThreddsController {
private void resetCallerToken() {
if(callerToken!=null) {
log.trace("Resetting caller token {}. Target Token is {}, current is {} ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken()));
log.trace(String.format("Resetting caller token %1$s. Target Token is %2$s, current is %3$s ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())));
TokenSetter.setToken(callerToken);
callerToken=null;
}else log.trace("Caller token {} already reset [current token {}]. Target Token is {}",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken));
}else log.trace(String.format("Caller token %1$s already reset [current token %2$s]. Target Token is %3$s",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)));
}
public final ThreddsInfo getThreddsInfo() {
@ -208,6 +210,7 @@ public class ThreddsController {
public ThreddsCatalog createCatalog(String name) throws InternalException {
setTargetToken();
try{
SecurityTokenProvider.instance.get();
log.info("Creating catalog with name {} for path {} ",name,operatingPath);
String sdiUrl="http://"+getSDIServiceHost()+"/"+Constants.SDI_THREDDS_BASE_URL;
Response resp=getWebClient().target(sdiUrl).
@ -267,7 +270,7 @@ public class ThreddsController {
}
public TransferResult transferFile(Destination dest,String url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
public TransferResult transferFile(Destination dest,URL url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
setTargetToken();
try{DataTransferClient client=getDTClient(hostname);
if(invocations!=null&&!invocations.isEmpty())

119
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java
View File

@ -1,20 +1,17 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.io.File;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
import org.gcube.usecases.ws.thredds.Constants;
@ -43,7 +40,7 @@ public class WorkspaceFolderManager {
}
private WorkspaceFolder theFolder;
private FolderContainer theFolder;
private String folderId;
@ -52,20 +49,21 @@ public class WorkspaceFolderManager {
private SynchFolderConfiguration config=null;
private ThreddsController threddsController=null;
private Workspace ws;
private StorageHubClient sc;
public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException {
try{
ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
theFolder=(WorkspaceFolder) ws.getItem(folderId);
// ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
sc=WorkspaceUtils.getClient();
theFolder=sc.open(folderId).asFolder();
this.folderId=folderId;
}catch(WorkspaceException | InternalErrorException | HomeNotFoundException | UserNotFoundException e) {
}catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to access folder id "+folderId,e);
}
}
public WorkspaceFolder getTheFolder() {
public FolderContainer getTheFolder() {
return theFolder;
}
@ -79,21 +77,17 @@ public class WorkspaceFolderManager {
private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
try {
WorkspaceFolder root=(WorkspaceFolder) ws.getItem(getSynchConfiguration().getRootFolderId());
FolderContainer root=sc.open(getSynchConfiguration().getRootFolderId()).asFolder();
SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root);
return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken());
}catch(WorkspaceException | InternalErrorException e) {
}catch(StorageHubException e) {
throw new WorkspaceInteractionException(e);
}
}
public boolean isRoot() throws WorkspaceNotSynchedException, WorkspaceInteractionException{
try{
return getSynchConfiguration().getRootFolderId().equals(theFolder.getId());
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException(e);
}
return getSynchConfiguration().getRootFolderId().equals(theFolder.getId());
}
@ -103,20 +97,16 @@ public class WorkspaceFolderManager {
if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched.");
log.debug("Loading properties for ");
config=WorkspaceUtils.loadConfiguration(theFolder);
}catch(InternalErrorException e) {
}catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e);
}
}
return config;
}
public boolean isSynched() throws WorkspaceInteractionException {
try{
Map<String,String> props=theFolder.getProperties().getProperties();
public boolean isSynched(){
Map<String,Object> props=theFolder.get().getMetadata().getMap();
return props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null);
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException("Unable to check Synch flag on "+folderId,e);
}
}
@ -126,9 +116,9 @@ public class WorkspaceFolderManager {
SynchFolderConfiguration config=getSynchConfiguration();
try{
checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder));
return new SyncFolderDescriptor(this.folderId,this.theFolder.getPath(),config);
}catch(InternalErrorException e) {
checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder.get()));
return new SyncFolderDescriptor(this.folderId,this.theFolder.get().getPath(),config);
}catch(StorageHubException e) {
throw new WorkspaceInteractionException(e);
}
}
@ -188,7 +178,7 @@ public class WorkspaceFolderManager {
}catch(InternalException e) {
throw new InternalException ("Unable to check/initialize remote folder",e);
}catch(InternalErrorException e) {
}catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to set Properties to "+folderId,e);
}
}
@ -202,12 +192,12 @@ public class WorkspaceFolderManager {
WorkspaceUtils.cleanItem(theFolder);
if(deleteRemote)
getThreddsController().createEmptyFolder(null);
}catch(InternalErrorException e) {
}catch(StorageHubException e) {
throw new WorkspaceInteractionException("Unable to cleanup "+folderId,e);
}
}
public void setLastUpdateTime() throws InternalErrorException {
public void setLastUpdateTime() throws StorageHubException {
WorkspaceUtils.setLastUpdateTime(theFolder, System.currentTimeMillis());
}
@ -251,12 +241,12 @@ public class WorkspaceFolderManager {
private static void checkFolder(WorkspaceFolder folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws InternalErrorException, InternalException {
private static void checkFolder(FolderContainer folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws StorageHubException, InternalException {
// Check folder configuration
log.trace("Checking folder {} ",folder.getPath());
log.trace("Checking folder {} ",folder.get().getPath());
log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder);
String folderName=folder.getName();
String folderName=folder.get().getName();
String currentRemotePath=rootConfig.getRemotePath()+((relativePathFromRootFolder==null)?"":"/"+relativePathFromRootFolder);
@ -270,19 +260,19 @@ public class WorkspaceFolderManager {
log.debug("Initializing properties for {} ",folderName);
//INIT PROPERTIES IF NOT PRESENT
if(!WorkspaceUtils.isConfigured(folder))
if(!WorkspaceUtils.isConfigured(folder.get()))
WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
for(WorkspaceItem item:folder.getChildren()) {
String itemName=item.getName();
for(ItemContainer<?> item:folder.list().withAccounting().withMetadata().getContainers()) {
String itemName=item.get().getName();
String itemRelativePath=(relativePathFromRootFolder==null)?itemName:relativePathFromRootFolder+"/"+itemName;
String itemRemotePath=currentRemotePath+"/"+itemName;
if(item.isFolder()) {
if(item.getType().equals(ContainerType.FOLDER)) {
if(recursive)
checkFolder((WorkspaceFolder) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine);
checkFolder((FolderContainer) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine);
else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
}else if(rootConfig.matchesFilter(itemName)) {
if(!WorkspaceUtils.isConfigured(item))
if(!WorkspaceUtils.isConfigured(item.get()))
WorkspaceUtils.initProperties(item, null, null, null,null,null,null);
}
currentFolderExistingItem.add(itemName);
@ -305,31 +295,50 @@ public class WorkspaceFolderManager {
currentFolderExistingItem, remoteFolderItems,
controller, null, null);
if(accountingEntries.isEmpty()) {
log.debug("No accounting entries found");
log.debug("No Accounting Entries to be managed..");
folderStatus=SynchronizationStatus.UP_TO_DATE;
}
// CHECK WS ITEMS
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder()||rootConfig.matchesFilter(item.getName())) {
SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item, remoteFolderItems, controller,lastUpdatedRoutine);
item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+""));
for(ItemContainer<?> item:folder.list().withAccounting().withMetadata().getContainers())
if(item.getType().equals(ContainerType.FOLDER)||rootConfig.matchesFilter(item.get().getName())) {
SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item.get(), remoteFolderItems, controller,lastUpdatedRoutine);
Metadata meta=item.get().getMetadata();
Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+"");
meta.setMap(map);
item.setMetadata(meta);
folderStatus=folderStatus.equals(SynchronizationStatus.UP_TO_DATE)?itemStatus:folderStatus;
}
}
// CHECK REMOTE FOLDER
if(folderStatus.equals(SynchronizationStatus.UP_TO_DATE)) {
Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, accountingEntries, currentFolderExistingItem, folder, controller, rootConfig, null, null);
if(!toImportItems.isEmpty()) folderStatus=SynchronizationStatus.OUTDATED_WS;
}
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+""));
Metadata meta=folder.get().getMetadata();
Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+"");
meta.setMap(map);
folder.setMetadata(meta);
}else {
// Remote Folder not existing, set everything to OUTDATED_REMOTE
for(WorkspaceItem item:folder.getChildren())
item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""));
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""));
for(ItemContainer<?> item:folder.list().withMetadata().getContainers()) {
Metadata meta=item.get().getMetadata();
Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"");
meta.setMap(map);
item.setMetadata(meta);
}
Metadata meta=folder.get().getMetadata();
Map<String,Object> map=meta.getMap();
map.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"");
meta.setMap(map);
folder.setMetadata(meta);
}

262
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java
View File

@ -1,26 +1,31 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibary.model.items.type.WorkspaceItemType;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.Properties;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRemoval;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRenaming;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.nodes.accounting.AccountEntry;
import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRemoval;
import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEntryRenaming;
import org.gcube.common.storagehub.model.types.WorkspaceItemType;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -33,10 +38,12 @@ public class WorkspaceUtils {
public static StorageHubClient getClient() throws StorageHubException{
return new StorageHubClient();
}
/**
* Checks current accounting info in order to infere synchronization status.
* Checks current accounting info in order to infer synchronization status.
* OwnerProcess and service parameters can be null for check purposes.
*
* @param folderPath
@ -52,53 +59,62 @@ public class WorkspaceUtils {
* @throws InternalErrorException
*/
static Set<String> scanAccountingForStatus(
WorkspaceFolder toScanFolder,
FolderContainer toScanFolder,
SynchFolderConfiguration config,
Set<String> localChildrenNames,
Set<String> remoteChildrenNames,
ThreddsController folderController,
Process ownerProcess,
ExecutorService service) throws InternalErrorException{
ExecutorService service) throws StorageHubException{
Set<String> handledAccountingEntries=new HashSet<>();
log.debug("Checking history of {} ",toScanFolder.getPath());
FolderItem toScanFolderItem=toScanFolder.get();
log.debug("Checking history of {} ",toScanFolderItem.getPath());
String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
String relativePath=toScanFolderItem.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
Date folderLastUpdateTime=null;
try{
folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolder);
folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolderItem);
}catch(Throwable t) {
log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolder.getName(),t);
log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolderItem.getName(),t);
folderLastUpdateTime=new Date(0l);
}
// scanning for deletions
log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolder.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime));
for(AccountingEntry entry:toScanFolder.getAccounting()) {
log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolderItem.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime));
for(AccountEntry entry:toScanFolderItem.getAccounting().getEntries()) {
try {
Date eventTime=entry.getDate().getTime();
if(folderLastUpdateTime==null|| eventTime.after(folderLastUpdateTime)) { // SKIP IF ENTRY OLDER THAN LAST UPDATE TIME
String toDeleteRemote=null;
switch(entry.getEntryType()) {
switch(entry.getType()) {
case CUT:
case REMOVAL:{
AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry;
if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER)||
AccountFolderEntryRemoval removalEntry=(AccountFolderEntryRemoval) entry;
if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER.toString())||
config.matchesFilter(removalEntry.getItemName()))
toDeleteRemote=removalEntry.getItemName();
break;
}
case RENAMING:{
AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry;
WorkspaceItem newItem=toScanFolder.find(renamingEntry.getNewItemName());
if(newItem.isFolder()||config.matchesFilter(renamingEntry.getOldItemName()))
case RENAMING:{
AccountFolderEntryRenaming renamingEntry=(AccountFolderEntryRenaming) entry;
ItemContainer<?> newItem=toScanFolder.findByName(renamingEntry.getNewItemName()).getContainers().get(0);
if(newItem.getType().equals(ContainerType.FOLDER)||config.matchesFilter(renamingEntry.getOldItemName()))
toDeleteRemote=renamingEntry.getOldItemName();
break;
}
}
default : {
log.debug("Skpping accounting entry {} ",entry.getType());
}
}
@ -117,7 +133,7 @@ public class WorkspaceUtils {
if(service!=null) {
log.debug("Service is not null. Submitting request ... ");
if(eventTime.after(remoteDate)) {
service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolder,toDeleteRemote)));
service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolderItem,toDeleteRemote)));
handledAccountingEntries.add(toDeleteRemote);
log.debug("Submitted DELETION request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
}
@ -161,13 +177,15 @@ public class WorkspaceUtils {
RemoteFileDescriptor folderDesc,
Set<String> handledAccountingEntries,
Set<String> handledWorkspaceItemEntries,
WorkspaceFolder toScanFolder,
FolderContainer toScanFolder,
ThreddsController folderController,
SynchFolderConfiguration config,
Process ownerProcess,
ExecutorService service) throws InternalException, InternalErrorException{
ExecutorService service) throws InternalException, StorageHubException{
log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanFolder.getPath(),folderDesc.getAbsolutePath());
FolderItem toScanItem=toScanFolder.get();
log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanItem.getPath(),folderDesc.getAbsolutePath());
Set<String> handledRemoteElements=new HashSet<String>();
// String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
@ -189,7 +207,7 @@ public class WorkspaceUtils {
log.debug("Child {} matches filter...");
handledRemoteElements.add(child);
if(service!=null) {
service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, child)));
service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanItem, child)));
log.debug("Submitted IMPORT request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
}
// import if matching
@ -201,10 +219,12 @@ public class WorkspaceUtils {
}
static void initProperties(WorkspaceItem toInit, String remotePath, String filter, String targetToken,
String catalogName,Boolean validateMeta, String rootFolderId) throws InternalErrorException {
static void initProperties(ItemContainer<?> toInit, String remotePath, String filter, String targetToken,
String catalogName,Boolean validateMeta, String rootFolderId) throws StorageHubException {
Map<String,String> toSetProperties=toInit.getProperties().getProperties();
Metadata meta=toInit.get().getMetadata();
Map<String,Object> toSetProperties=meta.getMap();
initIfMissing(toSetProperties,Constants.WorkspaceProperties.TBS,"true");
@ -212,7 +232,7 @@ public class WorkspaceUtils {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_STATUS,StepReport.Status.OK+"");
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.UP_TO_DATE+"");
if(toInit.isFolder()) {
if(toInit.getType().equals(ContainerType.FOLDER)) {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE);
@ -222,26 +242,28 @@ public class WorkspaceUtils {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId);
}else {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.METADATA_UUID,null);
}
toInit.getProperties().addProperties(toSetProperties);
}
meta.setMap(toSetProperties);
toInit.setMetadata(meta);
}
private static void initIfMissing(Map<String,String> current,String key,String defaultValue) {
private static void initIfMissing(Map<String,Object> current,String key,String defaultValue) {
if(!current.containsKey(key)||
current.get(key)==null||
current.get(key).equals("null")) current.put(key, defaultValue);
}
static boolean isConfigured(WorkspaceItem toCheck) throws InternalErrorException {
return isConfigured(toCheck.getProperties().getProperties());
static boolean isConfigured(Item toCheck) throws StorageHubException {
return isConfigured(toCheck.getMetadata().getMap());
}
static boolean isConfigured(Map<String,String> toCheckProperties) {
static boolean isConfigured(Map<String,Object> toCheckProperties) {
return (toCheckProperties.containsKey(Constants.WorkspaceProperties.TBS)&&toCheckProperties.get(Constants.WorkspaceProperties.TBS)!=null);
}
static SynchronizationStatus getStatusAgainstRemote(WorkspaceItem item, Set<String> existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException, InternalErrorException, RemoteFileNotFoundException {
static SynchronizationStatus getStatusAgainstRemote(Item item, Set<String> existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException,RemoteFileNotFoundException {
String itemName=item.getName();
SynchronizationStatus status=SynchronizationStatus.OUTDATED_REMOTE;
if(existingRemote.contains(itemName)) {
@ -269,88 +291,132 @@ public class WorkspaceUtils {
}
//
// /**
// *
// * @return max date between creation time, last modification time && LAST-UPDATE-PROP
// * @throws InternalErrorException
// * @throws NumberFormatException
// */
// static Date getMaxLastUpdate(WorkspaceItem item) throws NumberFormatException, InternalErrorException {
// return new Date(Long.max(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)),item.getLastModificationTime().getTimeInMillis()));
// }
static Date safelyGetLastUpdate(WorkspaceItem item) throws InternalErrorException {
static Date safelyGetLastUpdate(Item item){
try {
return new Date(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)));
return new Date(Long.parseLong(item.getMetadata().getMap().get(Constants.WorkspaceProperties.LAST_UPDATE_TIME)+""));
}catch(NumberFormatException e) {
log.debug("Unable to get last update time for {} ",item.getName(),e);
return new Date(0l);
}
}
public static boolean isModifiedAfter(WorkspaceItem item,Date fromDate) throws InternalErrorException {
for(AccountingEntry entry:item.getAccounting()) {
if(entry.getDate().getTime().after(fromDate)) {
switch(entry.getEntryType()) {
case PASTE:
case CREATE:
case RESTORE:
case UPDATE:
case ADD: return true;
}
}
}
return false;
public static boolean isModifiedAfter(Item item,Date fromDate) {
return item.getLastModificationTime().after(fromDate);
// for(AccountingEntry entry:item.getAccounting()) {
// if(entry.getDate().getTime().after(fromDate)) {
// switch(entry.getEntryType()) {
// case PASTE:
// case CREATE:
// case RESTORE:
// case UPDATE:
// case ADD: return true;
// }
// }
// }
// return false;
}
static void cleanItem(WorkspaceItem item) throws InternalErrorException {
Properties props=item.getProperties();
if(props.hasProperty(Constants.WorkspaceProperties.TBS)) {
if(item.isFolder()) {
props.addProperties(Constants.cleanedFolderPropertiesMap);
for(WorkspaceItem child : ((WorkspaceFolder)item).getChildren())
static void cleanItem(ItemContainer<?> itemContainer) throws StorageHubException{
Metadata meta=itemContainer.get().getMetadata();
Map<String,Object> map=meta.getMap();
if(map.containsKey(Constants.WorkspaceProperties.TBS)) {
if(itemContainer.getType().equals(ContainerType.FOLDER)) {
map=Constants.cleanedFolderPropertiesMap;
for(ItemContainer<?> child : ((FolderContainer)itemContainer).list().withMetadata().getContainers())
cleanItem(child);
}else props.addProperties(Constants.cleanedItemPropertiesMap);
}else map=Constants.cleanedItemPropertiesMap;
//Actually posting cleaned map
meta.setMap(map);
log.debug("Setting meta to item "+itemContainer.getId()+" : "+itemContainer.get().getPath());
itemContainer.setMetadata(meta);
}
}
static void setLastUpdateTime(WorkspaceFolder folder,long toSetTime) throws InternalErrorException {
StepReport.Status currentWSStatus=StepReport.Status.valueOf(folder.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_STATUS));
static void setLastUpdateTime(FolderContainer folder,long toSetTime) throws StorageHubException {
Metadata meta=folder.get().getMetadata();
Map<String,Object> map=meta.getMap();
StepReport.Status currentWSStatus=StepReport.Status.valueOf(map.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+"");
if(currentWSStatus.equals(StepReport.Status.OK))
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+""));
map.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+"");
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder()) setLastUpdateTime((WorkspaceFolder) item, toSetTime);
//Actually posting cleaned map
meta.setMap(map);
folder.setMetadata(meta);
// for(ItemContainer<?> child : folder.list().withMetadata().getContainers())
// if(child.getType().equals(ContainerType.FOLDER))
// setLastUpdateTime((FolderContainer) child, toSetTime);
}
public static SynchFolderConfiguration loadConfiguration(WorkspaceItem item) throws InternalErrorException {
if(item.isFolder()) {
Properties props=item.getProperties();
public static SynchFolderConfiguration loadConfiguration(ItemContainer<?> item) throws StorageHubException {
if(item.getType().equals(ContainerType.FOLDER)) {
Map<String,Object> map=item.get().getMetadata().getMap();
SynchFolderConfiguration config=new SynchFolderConfiguration();
config.setFilter(props.getPropertyValue(Constants.WorkspaceProperties.SYNCH_FILTER));
config.setRemotePath(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH));
config.setRemotePersistence(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
config.setTargetToken(props.getPropertyValue(Constants.WorkspaceProperties.TARGET_TOKEN));
config.setToCreateCatalogName(props.getPropertyValue(Constants.WorkspaceProperties.RELATED_CATALOG));
config.setValidateMetadata(Boolean.parseBoolean(props.getPropertyValue(Constants.WorkspaceProperties.VALIDATE_METADATA)));
config.setRootFolderId(props.getPropertyValue(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
config.setFilter(""+map.get(Constants.WorkspaceProperties.SYNCH_FILTER));
config.setRemotePath(""+map.get(Constants.WorkspaceProperties.REMOTE_PATH));
config.setRemotePersistence(""+map.get(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
config.setTargetToken(""+map.get(Constants.WorkspaceProperties.TARGET_TOKEN));
config.setToCreateCatalogName(""+map.get(Constants.WorkspaceProperties.RELATED_CATALOG));
config.setValidateMetadata(Boolean.parseBoolean(""+map.get(Constants.WorkspaceProperties.VALIDATE_METADATA)));
config.setRootFolderId(""+map.get(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
return config;
}else return loadConfiguration(item.getParent());
}else {
FolderContainer parentFolder=getClient().open(item.get().getParentId()).asFolder();
return loadConfiguration(parentFolder);
}
}
static void resetStatus(WorkspaceItem item) throws InternalErrorException {
if(item.isFolder()) {
for(WorkspaceItem child: ((WorkspaceFolder)item).getChildren())
static void resetStatus(ItemContainer<?> item) throws StorageHubException {
if(item.getType().equals(ContainerType.FOLDER)) {
for(ItemContainer<?> child : ((FolderContainer)item).list().withMetadata().getContainers())
resetStatus(child);
}
Map<String,String> props=item.getProperties().getProperties();
if(props.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) {
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+"");
item.getProperties().addProperties(props);
Metadata meta=item.get().getMetadata();
Map<String,Object> map=meta.getMap();
if(map.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) {
map.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+"");
//Actually posting cleaned map
meta.setMap(map);
item.setMetadata(meta);
}
}
public static void addParameters(ItemContainer<?> item, Map<String,Object> toAdd) throws StorageHubException {
Metadata meta=item.get().getMetadata();
Map<String,Object> current=meta.getMap();
current.putAll(toAdd);
meta.setMap(current);
item.setMetadata(meta);
}
public static void addParameter(ItemContainer<?> item, String key, Object value) throws StorageHubException {
Metadata meta=item.get().getMetadata();
Map<String,Object> current=meta.getMap();
current.put(key,value);
meta.setMap(current);
item.setMetadata(meta);
}
public static ItemContainer<?> scan(FolderContainer folder,String path) throws ItemNotFoundException, StorageHubException{
String toLookFor=path.substring((path.startsWith("/")?1:0), path.length());
String[] split=toLookFor.split("/");
toLookFor=split[0];
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.get().getName().equals(toLookFor)) {
if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length()));
else return item;
}
throw new ItemNotFoundException("Unable to find "+path);
}
}

4
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java
View File

@ -1,6 +1,6 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
@ -10,7 +10,7 @@ public class DeleteRemoteRequest extends SynchronizationRequest {
private String toRemoveName;
public DeleteRemoteRequest(Process process,WorkspaceFolder location,String name) {
public DeleteRemoteRequest(Process process,FolderItem location,String name) {
super(process,location);
this.toRemoveName=name;
}

4
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java
View File

@ -2,7 +2,7 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
@ -45,7 +45,7 @@ public class ProcessInitializationThread implements Runnable {
} catch (InternalException e) {
log.error("Unable to proceed..",e);
theProcess.cancel();
} catch (InternalErrorException e) {
} catch (StorageHubException e) {
log.error("Unable to proceed..",e);
theProcess.cancel();
}catch(Throwable t) {

31
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java
View File

@ -3,10 +3,12 @@ package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.io.BufferedWriter;
import java.io.FileWriter;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.SyncEngine;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@ -81,33 +83,30 @@ public class RequestLogger {
}
private static final String getRemotePath(WorkspaceItem item) {
private static final String getRemotePath(Item item) {
try{
if(item.isFolder()) return item.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
else return getRemotePath(item.getParent());
}catch(InternalErrorException e) {
if(item instanceof FolderItem) return item.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get());
}catch(StorageHubException e) {
log.warn("Unable to get Remote Path ",e);
return "N/A";
}
}
private static final String getName(WorkspaceItem item) {
try {
private static final String getName(Item item) {
return item.getName();
}catch(InternalErrorException e) {
log.warn("Unable to get name ",e);
return "N/A";
}
}
private static final String getWSPath(WorkspaceItem item) {
private static final String getWSPath(Item item) {
try{
if(item.isFolder()) return item.getPath();
else return getRemotePath(item.getParent());
}catch(InternalErrorException e) {
if(item instanceof FolderItem) return item.getPath();
else return getRemotePath(WorkspaceUtils.getClient().open(item.getParentId()).asItem().get());
}catch(StorageHubException e) {
log.warn("Unable to get WS Path ",e);
return "N/A";
}
}
}

4
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java
View File

@ -1,6 +1,6 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
@ -12,5 +12,5 @@ public abstract class SynchronizationRequest {
@NonNull
private Process process;
@NonNull
private WorkspaceFolder location;
private FolderItem location;
}

129
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java
View File

@ -8,9 +8,15 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
@ -32,7 +38,6 @@ import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.CancellationException;
import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.StepReport.OperationType;
@ -60,22 +65,25 @@ public class SynchronizationThread implements Runnable {
RequestLogger.get().log(theRequest);
String reportMessage="Never started";
String reportItemName="Still Unknown";
Status toSetStatus=Status.ERROR;
WorkspaceFolder parentFolder=theRequest.getLocation();
Status toSetStatus=Status.ERROR;
FolderItem parentFolderItem=theRequest.getLocation();
try {
StorageHubClient client=WorkspaceUtils.getClient();
FolderContainer parentFolder=client.open(parentFolderItem.getId()).asFolder();
checkCancelledProcess();
SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(theRequest.getLocation());
SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(parentFolder);
ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken());
if(theRequest instanceof TransferToThreddsRequest) {
TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest;
WorkspaceItem item=request.getToTransfer();
Item item=request.getToTransfer();
FileContainer itemContainer=client.open(item.getId()).asFile();
//look for metadata in same folder
String itemName=item.getName();
reportItemName=itemName;
String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml";
WorkspaceItem metadataItem=getFileByName(item.getParent(),false,toLookMetadataName);
FileContainer metadataItem=getFileByName(client.open(item.getParentId()).asFolder(),false,toLookMetadataName);
// if not present, generate with sis/geotk
Destination toSetDestination=new Destination();
@ -86,7 +94,7 @@ public class SynchronizationThread implements Runnable {
toSetDestination.setPersistenceId(synchConfig.getRemotePersistence());
//NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
String fileLocation=request.getLocation().getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
String fileLocation=request.getLocation().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
toSetDestination.setSubFolder(fileLocation);
@ -108,17 +116,17 @@ public class SynchronizationThread implements Runnable {
checkCancelledProcess();
TransferResult result=controller.transferFile(toSetDestination, item.getPublicLink(false), invocations);
TransferResult result=controller.transferFile(toSetDestination, itemContainer.getPublicLink(), invocations);
Map<String,String> toSetProperties=new HashMap<String,String>();
Map<String,Object> toSetProperties=new HashMap<String,Object>();
String toSetMetadataUUID=null;
Boolean validateMetadata=synchConfig.getValidateMetadata();
checkCancelledProcess();
if(metadataItem==null) {
ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID);
@ -134,15 +142,18 @@ public class SynchronizationThread implements Runnable {
toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID);
toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+"");
item.getProperties().addProperties(toSetProperties);
org.gcube.common.storagehub.model.Metadata meta=item.getMetadata();
meta.setMap(toSetProperties);
item.setMetadata(meta);
reportMessage="Successfully transferred and published.";
// End ws->th
} else if(theRequest instanceof TransferFromThreddsRequest) {
Map<String,String> toSetProperties=new HashMap<String,String>();
Map<String,Object> toSetProperties=new HashMap<String,Object>();
TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest;
String toImportName=null;
WorkspaceItem targetItem=null;
FileContainer targetItem=null;
if(importRequest.getTargetItem()==null) {
//Target Item will be created
toImportName=importRequest.getRemoteFilename();
@ -150,29 +161,38 @@ public class SynchronizationThread implements Runnable {
}else {
//Target Item already exists
toImportName=importRequest.getTargetItem().getName();
targetItem=importRequest.getTargetItem();
targetItem=client.open(importRequest.getTargetItem().getId()).asFile();
reportMessage="Updating file..";
}
reportItemName=toImportName;
RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName);
toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+"");
InputStream source=null;
try {
source=controller.getInputStream(toImportName);
if(targetItem==null)
targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source);
else
targetItem.updateItem(source);
targetItem.getProperties().addProperties(toSetProperties);
targetItem=parentFolder.uploadFile(source, toImportName, "Imported from Thredds");
// if(targetItem==null)
// targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source);
// else {
// targetItem.updateItem(source);
// }
org.gcube.common.storagehub.model.Metadata meta=targetItem.get().getMetadata();
meta.setMap(toSetProperties);
targetItem.setMetadata(meta);
}finally {
if(source!=null)
source.close();
}
reportMessage="File successfully imported";
}else if(theRequest instanceof DeleteRemoteRequest) {
DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest;
reportItemName=deleteRequest.getToRemoveName();
@ -201,7 +221,7 @@ public class SynchronizationThread implements Runnable {
log.debug("Remote File not found ",e);
reportMessage="Remote File not found : "+e.getMessage();
toSetStatus=Status.ERROR;
}catch(InternalErrorException e) {
}catch(StorageHubException e) {
log.debug("Internal generic exception ",e);
reportMessage="Internal error : "+e.getMessage();
toSetStatus=Status.ERROR;
@ -210,7 +230,7 @@ public class SynchronizationThread implements Runnable {
reportMessage="Unexpected exception : "+t.getMessage();
toSetStatus=Status.ERROR;
}finally {
updateParentProperty(parentFolder, toSetStatus);
updateParentProperty(parentFolderItem, toSetStatus);
submitReport(reportItemName,reportMessage,toSetStatus);
ProcessIdProvider.instance.reset();
}
@ -218,19 +238,32 @@ public class SynchronizationThread implements Runnable {
@Synchronized
private static void updateParentProperty(WorkspaceFolder folder,StepReport.Status toSetStatus) {
try {
String currentValue=folder.getProperties().getProperties().get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS);
if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null"))
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""));
else {
StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue);
if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus))
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""));
private static void updateParentProperty(FolderItem folderItem,StepReport.Status toSetStatus) {
boolean retry=false;
do {
try {
retry=false;
FolderContainer folder=WorkspaceUtils.getClient().open(folderItem.getId()).asFolder();
org.gcube.common.storagehub.model.Metadata folderMeta=folder.get().getMetadata();
Map<String,Object> props=folderMeta.getMap();
String currentValue=props.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+"";
if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null"))
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
else {
StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue);
if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus))
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
}
folderMeta.setMap(props);
folder.setMetadata(folderMeta);
}catch(ItemLockedException e) {
log.debug("Item locked retrying.. ",e);
retry=true;
}catch(Throwable t) {
log.warn("Unable to update folder status ",t);
}
}catch(Throwable t) {
log.warn("Unable to update folder status ",t);
}
}while(retry);
}
@ -254,11 +287,11 @@ public class SynchronizationThread implements Runnable {
private static MetadataReport publishMetadata(WorkspaceItem toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{
private static MetadataReport publishMetadata(FileContainer toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{
File tempMetaFile=null;
try {
Metadata meta=SDIAbstractPlugin.metadata().build();
tempMetaFile=NetUtils.download(toPublish.getPublicLink(false));
tempMetaFile=NetUtils.download(toPublish.getPublicLink());
log.debug("Publishing metadata {} ",filename);
@ -274,14 +307,14 @@ public class SynchronizationThread implements Runnable {
}
private static final WorkspaceItem getFileByName(WorkspaceFolder toLookIntoFolder,boolean recursive,String toLookForName) throws InternalErrorException {
log.debug("Looking for {} into {} [recursive {} ]",toLookForName,toLookIntoFolder.getPath()+" ID "+toLookIntoFolder.getId(),recursive);
for(WorkspaceItem item : toLookIntoFolder.getChildren())
if(!item.isFolder()&&item.getName().equals(toLookForName)) return item;
private static final FileContainer getFileByName(FolderContainer toLookIntoFolder,boolean recursive,String toLookForName) throws StorageHubException {
log.debug(String.format("Looking for %1$s into %2$s [recursive %3$s ]",toLookForName,toLookIntoFolder.get().getPath()+" ID "+toLookIntoFolder.getId(),recursive));
for(ItemContainer<?> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
if(!item.getType().equals(ContainerType.FOLDER)&&item.get().getName().equals(toLookForName)) return (FileContainer) item;
if(recursive) {
for(WorkspaceItem item : toLookIntoFolder.getChildren())
if(item.isFolder()) return getFileByName((WorkspaceFolder) item, recursive, toLookForName);
for(ItemContainer<?> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
if(item.getType().equals(ContainerType.FOLDER)) return getFileByName((FolderContainer) item, recursive, toLookForName);
}
return null;

17
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java
View File

@ -1,23 +1,32 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Data
@Slf4j
public class TransferFromThreddsRequest extends SynchronizationRequest {
private WorkspaceItem targetItem; // can be null
private Item targetItem; // can be null
private String remoteFilename;
public TransferFromThreddsRequest(Process process, WorkspaceItem targetItem, WorkspaceFolder containingFolder,
public TransferFromThreddsRequest(Process process, Item targetItem, FolderItem containingFolder,
String remoteFilename) {
super(process,containingFolder);
this.targetItem = targetItem;
this.remoteFilename = remoteFilename;
if(targetItem==null)
log.debug(String.format("Created IMPORT request of %1$s into %1$s from %3$s",
remoteFilename,containingFolder.getPath(),containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
else
log.debug(String.format("Created UPDATE %1$s from %2$s", targetItem.getPath(),
containingFolder.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
}

14
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java
View File

@ -1,23 +1,27 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Getter
@Setter
@Slf4j
public class TransferToThreddsRequest extends SynchronizationRequest {
private WorkspaceItem toTransfer;
private Item toTransfer;
public TransferToThreddsRequest(Process process,WorkspaceFolder location, WorkspaceItem toTransfer) {
public TransferToThreddsRequest(Process process,FolderItem location, Item toTransfer) {
super(process,location);
this.toTransfer = toTransfer;
log.debug(String.format("Created SEND request from $1%s to $2%s",
toTransfer.getPath(),location.getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)));
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/ItemNotFoundException.java Normal file
View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class ItemNotFoundException extends WorkspaceInteractionException {
/**
*
*/
private static final long serialVersionUID = -6818076224344812668L;
public ItemNotFoundException() {
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public ItemNotFoundException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

18
src/main/java/org/gcube/usecases/ws/thredds/model/SyncEngineStatusDescriptor.java Normal file
View File

@ -0,0 +1,18 @@
package org.gcube.usecases.ws.thredds.model;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.ToString;
@Getter
@AllArgsConstructor
@ToString
public class SyncEngineStatusDescriptor {
private int parallelTransferCount;
private int queueLenght;
private Map<String,String> configuration;
}

4
src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties
View File

@ -1,6 +1,6 @@
scanner.pool.maxSize=10
scanner.pool.coreSize=1
scanner.pool.coreSize=10
scanner.pool.idle.ms=30000
transfers.pool.maxSize=40
transfers.pool.coreSize=1
transfers.pool.coreSize=20
transfers.pool.idle.ms=30000

40
src/test/java/org/gcube/usecases/ws/thredds/DTTests.java
View File

@ -1,40 +1,12 @@
package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import org.apache.commons.io.IOUtils;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class DTTests {
public static void main(String[] args) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException, MalformedURLException, FileNotFoundException, IOException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
public static void main(String[] args) throws Exception {
// TestCommons.setScope();
// String threddsHostname="thredds-d-d4s.d4science.org";
// DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+threddsHostname+":80");
@ -57,8 +29,10 @@ public class DTTests {
//
TokenSetter.set("/gcube/devNext");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
String folderId=ws.getItemByPath("/Workspace/Accounting").getId();
// Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
StorageHubClient client=WorkspaceUtils.getClient();
String folderId=TestCommons.getByPath("/Workspace/Accounting").getId();
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId);
System.out.println("Done");

37
src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java
View File

@ -1,44 +1,35 @@
package org.gcube.usecases.ws.thredds;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.homelibrary.home.workspace.folder.FolderItem;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class GetWSSynchSpace {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
TokenSetter.set("/gcube/preprod/preVRE");
public static void main(String[] args) throws Exception {
TokenSetter.set("/gcube");
SyncEngine engine=SyncEngine.get();
String folderId=TestCommons.getWSIdByPath("/Workspace/Thredds main catalog");
FolderContainer folder=(FolderContainer) TestCommons.getByPath("ThreddsDev");
System.out.println(folderId);
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
WorkspaceFolder folder=(WorkspaceFolder) ws.getItem(folderId);
System.out.println(folder.getId());
SynchFolderConfiguration config=new SynchFolderConfiguration();
System.out.println("Total size : "+computeLength(folder, config));
}
private static final long computeLength(WorkspaceItem item,SynchFolderConfiguration config) throws InternalErrorException {
private static final long computeLength(ItemContainer<?> item,SynchFolderConfiguration config) throws StorageHubException{
long toReturn=0l;
if(item.isFolder()) {
for(WorkspaceItem child:((WorkspaceFolder)item).getChildren()) {
if(item.isFolder()||config.matchesFilter(item.getName()))
if(item.getType().equals(ContainerType.FOLDER)) {
for(ItemContainer<?> child:((FolderContainer)item).list().withContent().getContainers()) {
if(child.getType().equals(ContainerType.FOLDER)||config.matchesFilter(child.get().getName()))
toReturn=toReturn+computeLength(child, config);
}
}else toReturn=toReturn+((FolderItem)item).getLength();
}else toReturn=toReturn+((FileContainer)item).get().getContent().getSize();
return toReturn;
}

56
src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java
View File

@ -4,23 +4,14 @@ import java.io.File;
import java.util.Map;
import java.util.Map.Entry;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Properties;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
@ -28,16 +19,17 @@ import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class PublicLinkIssueTest {
public static void main (String[] args) throws ItemNotFoundException, InternalErrorException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
public static void main (String[] args) throws Exception {
TokenSetter.set("/d4science.research-infrastructures.eu");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
System.out.println(ws.getItemByPath("/Workspace/ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc").getPublicLink(false));
StorageHubClient shClient=WorkspaceUtils.getClient();
System.out.println(((FileContainer)TestCommons.getByPath("ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc")).getPublicLink());
String threddsHostName = Commons.getThreddsHost();
@ -51,7 +43,7 @@ public class PublicLinkIssueTest {
client.httpSource("http://data.d4science.org/V2drR2gxSFRTQlpLVC9nakozL29QcDdPR2U5UEVHYWRHbWJQNStIS0N6Yz0", dest);
scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d"));
scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder());
System.out.println("OK FIRST ...");
try {
Metadata meta=SDIAbstractPlugin.metadata().build();
@ -65,29 +57,29 @@ public class PublicLinkIssueTest {
e.printStackTrace();
}
System.out.println("CHECKING AGAIN");
scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d"));
scanForPrint(shClient.open("a8cd78d3-69e8-4d02-ac90-681b2d16d84d").asFolder());
}
public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException {
System.out.println("Folder "+folder.getPath());
printProperties(folder.getProperties());
public static void scanForPrint(FolderContainer folder) throws StorageHubException {
System.out.println("Folder "+folder.get().getPath());
printProperties(folder.get());
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
for(WorkspaceItem item:folder.getChildren())
if(!item.isFolder()&&config.matchesFilter(item.getName())) {
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) {
// System.out.println("ITEM "+item.getPath());
printProperties(item.getProperties());
printProperties(item.get());
}
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder())scanForPrint((WorkspaceFolder) item);
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item);
}
public static void printProperties(Properties prop) throws InternalErrorException {
Map<String,String> map=prop.getProperties();
public static void printProperties(Item toPrint){
Map<String,Object> map=toPrint.getMetadata().getMap();
// System.out.print("Properties : ..");
for(Entry<String,String> entry:map.entrySet()) {
for(Entry<String,Object> entry:map.entrySet()) {
if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {

66
src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java
View File

@ -1,21 +1,13 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Map;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
@ -35,9 +27,9 @@ public class TestCommons {
String toCreateCatalogName;
public String getFolderId() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
public String getFolderId() throws Exception {
if(folderId.startsWith("/"))
return getWSIdByPath(folderId);
return getByPath(folderId).getId();
else return folderId;
}
@ -45,26 +37,46 @@ public class TestCommons {
private static Map<String,TestSet> configs=new HashMap<>();
private static String toUseConfig="pre";
private static String toUseConfig="default";
static {
configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","",""));
// configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","",""));
// folderName="WS-Tests";
configs.put("default", new TestSet("Default Tests","/gcube/devsec/devVRE","/Workspace/ThreddsDev","public/netcdf","***REMOVED***","main"));
configs.put("simple", new TestSet("Simple label ","/gcube", "Test1","public/netcdf/simpleFolder","***REMOVED***","simple"));
configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
configs.put("pre", new TestSet("Default Tests","/gcube/preprod/preVRE","/Workspace/CMEMS","public/netcdf/CMEMS","***REMOVED***","main"));
// configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main"));
//
//
// configs.put("pre", new TestSet("Default Tests","/pred4s/preprod/preVRE","Thredds Synch","Fabio WS","91a8433b-9491-42c1-95e2-420245ef1943-980114272","SynchTestCatalog"));
}
public static String getWSIdByPath(String path) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
return ws.getItemByPath(path).getId();
public static ItemContainer<?> getByPath(String path) throws Exception {
StorageHubClient client = WorkspaceUtils.getClient();
FolderContainer root=client.getWSRoot();
return scan(root,path);
}
public static ItemContainer<?> scan(FolderContainer folder,String path) throws Exception{
String toLookFor=path.substring((path.startsWith("/")?1:0), path.length());
String[] split=toLookFor.split("/");
toLookFor=split[0];
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.get().getName().equals(toLookFor)) {
if(split.length>1) return scan((FolderContainer)item,path.substring(toLookFor.length()));
else return item;
}
throw new Exception("Unable to find "+path);
}
@ -73,13 +85,13 @@ public class TestCommons {
}
public static WorkspaceFolder getTestFolder() throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, IOException {
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
public static FolderContainer getTestFolder() throws Exception {
try{
return (WorkspaceFolder) ws.getItem(configs.get(toUseConfig).getFolderId());
return WorkspaceUtils.getClient().open(configs.get(toUseConfig).getFolderId()).asFolder();
}catch(Throwable t) {
// try to use path
return (WorkspaceFolder) ws.getItemByPath(configs.get(toUseConfig).getFolderId());
return (FolderContainer) getByPath(configs.get(toUseConfig).getFolderId());
}
@ -101,12 +113,12 @@ public class TestCommons {
}
public static ThreddsController getThreddsController() throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
public static ThreddsController getThreddsController() throws Exception {
SynchFolderConfiguration config=getSynchConfig();
return new ThreddsController(config.getRemotePath(), config.getTargetToken());
}
public static SynchFolderConfiguration getSynchConfig() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
public static SynchFolderConfiguration getSynchConfig() throws Exception {
TestSet set=configs.get(toUseConfig);
return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId());
}

8
src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java
View File

@ -1,18 +1,12 @@
package org.gcube.usecases.ws.thredds;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class ThreddsTests {
public static void main(String[] args) throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
public static void main(String[] args) throws Exception{
TestCommons.setScope();
SynchFolderConfiguration folderConfig=TestCommons.getSynchConfig();

31
src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java
View File

@ -1,40 +1,31 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WSTimes {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, InternalErrorException, HomeNotFoundException, UserNotFoundException, IOException {
public static void main(String[] args) throws Exception {
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
for(WorkspaceItem item : folder.getChildren()) {
FolderContainer folder=TestCommons.getTestFolder();
for(ItemContainer<?> item : folder.list().withMetadata().getContainers()) {
printDates(item);
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
}
item.getProperties().addProperty("Fake prop", "fake value");
WorkspaceUtils.addParameter(item,"Fake prop", "fake value");
printDates(item);
}
}
public static final void printDates(WorkspaceItem item) throws InternalErrorException {
System.out.println("ITEM : "+item.getName());
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getCreationTime().getTime()));
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getLastModificationTime().getTime()));
public static final void printDates(ItemContainer<?> item) {
System.out.println("ITEM : "+item.get().getName());
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getCreationTime().getTime()));
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.get().getLastModificationTime().getTime()));
}
}

16
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java
View File

@ -1,24 +1,14 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.util.Date;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WorkspaceAccounting {
public static void main(String[] args) throws InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, IOException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, NumberFormatException, ItemNotFoundException {
public static void main(String[] args) throws Exception {
TestCommons.setScope();
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
// WorkspaceFolder folder=ws.getRoot().createFolder("Accounting", "test purposes");
@ -68,7 +58,7 @@ public class WorkspaceAccounting {
System.out.println(WorkspaceUtils.isModifiedAfter(ws.getItemByPath(TestCommons.getTestFolder().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc"), new Date(0l)));
System.out.println(WorkspaceUtils.isModifiedAfter(TestCommons.getByPath(TestCommons.getTestFolder().get().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc").get(), new Date(0l)));
}

28
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java
View File

@ -1,28 +1,24 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.model.Metadata;
public class WorkspaceCleanup {
public static void main(String[] args) throws WorkspaceInteractionException, InternalException, InternalErrorException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, HomeNotFoundException, UserNotFoundException, IOException {
public static void main(String[] args) throws Exception{
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
FolderContainer folder=TestCommons.getTestFolder();
Metadata meta=folder.get().getMetadata();
meta.setMap(Constants.cleanedFolderPropertiesMap);
folder.setMetadata(meta);
SyncEngine engine=SyncEngine.get();
engine.forceUnlock(folder.getId());
engine.unsetSynchronizedFolder(folder.getId(), false);
// engine.unsetSynchronizedFolder(folder.getId(), false);
//
WorkspaceProperties.printProperties(folder);
}
}

25
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java
View File

@ -1,36 +1,23 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Collections;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WorkspaceLock {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, HomeNotFoundException, UserNotFoundException, WorkspaceInteractionException, InternalException, MalformedURLException, IOException {
public static void main(String[] args) throws Exception {
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
FolderContainer folder=TestCommons.getTestFolder();
WorkspaceFolderManager manager=new WorkspaceFolderManager(folder.getId());
String processID="mytest";
try {
System.out.println("Trying to cleanup, first.. ");
folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null));
System.out.println("FOLDER PROPERTIES : "+folder.getProperties().getProperties());
WorkspaceUtils.addParameters(folder, Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null));
System.out.println("FOLDER PROPERTIES : "+folder.get().getMetadata().getMap());
}catch(Throwable t) {
System.err.println("Dismiss error : ");
t.printStackTrace();

52
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java
View File

@ -1,35 +1,29 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Map;
import java.util.Map.Entry;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Properties;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceProperties {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, ItemNotFoundException, MalformedURLException, IOException, WorkspaceInteractionException, InternalException {
public static void main(String[] args) throws Exception{
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
FolderContainer folder=TestCommons.getTestFolder();
// System.out.println("Props : "+folder.get().getMetadata().getMap().size());
// printProperties(folder);
// System.out.println(new WorkspaceFolderManager(folder.getId()).isSynched());
SyncEngine.get().check(folder.getId(), true);
scanForPrint(folder);
// SyncEngine.get().shutDown();
// for(Workspace)
// printProperties(folder.getProperties());
@ -61,24 +55,24 @@ public class WorkspaceProperties {
}
public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException {
System.out.println("Folder "+folder.getPath());
printProperties(folder.getProperties());
public static void scanForPrint(FolderContainer folder) throws StorageHubException {
System.out.println("Folder "+folder.get().getPath());
printProperties(folder);
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
for(WorkspaceItem item:folder.getChildren())
if(!item.isFolder()&&config.matchesFilter(item.getName())) {
System.out.println("ITEM "+item.getPath());
printProperties(item.getProperties());
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(!item.getType().equals(ContainerType.FOLDER)&&config.matchesFilter(item.get().getName())) {
System.out.println("ITEM "+item.get().getPath());
printProperties(item);
}
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder())scanForPrint((WorkspaceFolder) item);
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item);
}
public static void printProperties(Properties prop) throws InternalErrorException {
Map<String,String> map=prop.getProperties();
public static void printProperties(ItemContainer<?> item) {
Map<String,Object> map=item.get().getMetadata().getMap();
System.out.print("Properties : ..");
for(Entry<String,String> entry:map.entrySet()) {
for(Entry<String,Object> entry:map.entrySet()) {
// if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {

43
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java
View File

@ -1,23 +1,10 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.concurrent.Semaphore;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
@ -25,18 +12,20 @@ import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceSynchronizationTest {
public static void main(String[] args) throws ProcessNotFoundException, InternalErrorException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, HomeNotFoundException, UserNotFoundException, MalformedURLException, IOException {
public static void main(String[] args) throws Exception {
// GET ENGINE : SINGLETON INSTANCE
SyncEngine engine=SyncEngine.get();
final SyncEngine engine=SyncEngine.get();
engine.setRequestLogger("requests.txt");
//TEST INFO...
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
FolderContainer folder=TestCommons.getTestFolder();
// FOLDER CONFIGURATION BEAN
SynchFolderConfiguration config=TestCommons.getSynchConfig();
// SynchFolderConfiguration config=null;
// try {
// //try to clean it up, first..
@ -49,38 +38,40 @@ public class WorkspaceSynchronizationTest {
// engine.unsetSynchronizedFolder(folder.getId(), false);
// }
String folderId=folder.getId();
// String folderId="15db7b0e-c215-41b7-b384-a1deda250b65";
try {
// WHEN OPENING A FOLDER, INVOKE CHECK TO UPDATE SYNCH STATUS
engine.check(folder.getId(), false);
engine.check(folderId, false);
}catch(WorkspaceNotSynchedException e) {
System.out.println("Folder not synched, configurin it..");
engine.setSynchronizedFolder(config, folder.getId());
engine.check(folder.getId(), false);
engine.setSynchronizedFolder(config, folderId);
engine.check(folderId, false);
}catch(WorkspaceLockedException e) {
System.out.println("Workspace locked, going to force unlock.."); // MAINLY FOR TEST PURPOSES, OR WHEN SOMETHIGN GOES WRONG.. USE CAUTIOUSLY
engine.forceUnlock(folder.getId());
engine.check(folder.getId(), false);
engine.forceUnlock(folderId);
engine.check(folderId, false);
}
// INVOKE SYNCHRONIZATION ON FOLDER
ProcessDescriptor descriptor=engine.doSync(folder.getId());
ProcessDescriptor descriptor=engine.doSync(folderId);
System.out.println("Obtained descriptor : "+descriptor);
Semaphore sem=new Semaphore(0);
// REGISTER CALLBACK TO MONITOR PROGRESS
engine.registerCallBack(folder.getId(), new SyncOperationCallBack() {
engine.registerCallBack(folderId, new SyncOperationCallBack() {
@Override
public void onStep(ProcessStatus status, ProcessDescriptor descriptor) {
System.out.println("ON STEP : "+status+" "+descriptor);
System.out.println("ENGINE STATUS : "+engine.getStatus());
System.out.println("LOG : \n"+ status.getLogBuilder().toString());
if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release();
}
@ -93,7 +84,7 @@ public class WorkspaceSynchronizationTest {
}
engine.check(folder.getId(), true);
engine.check(folderId, true);
// INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES