gCubeSystem
/
ws-thredds
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/spatial-data/ws-thredds@167619 82a268e6-3cf1-43bd-a215-b396298e98cf

Browse Source
Fabio Sinibaldi 6 years ago
commit 4c09cfeb84
76 changed files with 5314 additions and 0 deletions
Show Stats Download Patch File Download Diff File

36
.classpath
Unescape Escape View File

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

23
.project
Unescape Escape View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>ws-thredds</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

6
.settings/org.eclipse.core.resources.prefs
Unescape Escape View File

@ -0,0 +1,6 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8

5
.settings/org.eclipse.jdt.core.prefs
Unescape Escape View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.8

4
.settings/org.eclipse.m2e.core.prefs
Unescape Escape View File

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=false
version=1

1
distro/LICENSE
Unescape Escape View File

@ -0,0 +1 @@
${gcube.license}

65
distro/README
Unescape Escape View File

@ -0,0 +1,65 @@
The gCube System - ${name}
--------------------------------------------------
${description}
${gcube.description}
${gcube.funding}
Version
--------------------------------------------------
${version} (${buildDate})
Please see the file named "changelog.xml" in this directory for the release notes.
Authors
--------------------------------------------------
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
Maintainers
-----------
* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy).
Download information
--------------------------------------------------
Source code is available from SVN:
${scm.url}
Binaries can be downloaded from the gCube website:
${gcube.website}
Installation
--------------------------------------------------
Installation documentation is available on-line in the gCube Wiki:
${gcube.wikiRoot}/Tabular_Data_Manager
Documentation
--------------------------------------------------
Documentation is available on-line in the gCube Wiki:
${gcube.wikiRoot}/Tabular_Data_Manager
${gcube.wikiRoot}/Tabular_Data_Manager
Support
--------------------------------------------------
Bugs and support requests can be reported in the gCube issue tracking tool:
${gcube.issueTracking}
Licensing
--------------------------------------------------
This software is licensed under the terms you may find in the file named "LICENSE" in this directory.

8
distro/changelog.xml
Unescape Escape View File

@ -0,0 +1,8 @@
<ReleaseNotes>
<Changeset component="ws-thredds-0.0.1" date="2017-08-22">
<Change>First Release</Change>
</Changeset>
<Changeset component="ws-thredds-0.0.2" date="2017-11-19">
<Change>Cleanup folder fix</Change>
</Changeset>
</ReleaseNotes>

30
distro/descriptor.xml
Unescape Escape View File

@ -0,0 +1,30 @@
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>servicearchive</id>
<formats>
<format>tar.gz</format>
</formats>
<baseDirectory>/</baseDirectory>
<fileSets>
<fileSet>
<directory>${distroDirectory}</directory>
<outputDirectory>/</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
<includes>
<include>README</include>
<include>LICENSE</include>
<include>changelog.xml</include>
</includes>
<fileMode>755</fileMode>
<filtered>true</filtered>
</fileSet>
</fileSets>
<files>
<file>
<source>target/${build.finalName}.${project.packaging}</source>
<outputDirectory>/${artifactId}</outputDirectory>
</file>
</files>
</assembly>

29
distro/profile.xml
Unescape Escape View File

@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<Resource>
<ID></ID>
<Type>Service</Type>
<Profile>
<Description>${description}</Description>
<Class>Portlets</Class>
<Name>${artifactId}</Name>
<Version>1.0.0</Version>
<Packages>
<Software>
<Description>${description}</Description>
<Name>${artifactId}</Name>
<Version>${version}</Version>
<MavenCoordinates>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
</MavenCoordinates>
<Type>library</Type>
<Files>
<File>${build.finalName}.jar</File>
</Files>
</Software>
</Packages>
</Profile>
</Resource>

69
pom.xml
Unescape Escape View File

@ -0,0 +1,69 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.gcube.tools</groupId>
<artifactId>maven-parent</artifactId>
<version>LATEST</version>
</parent>
<groupId>org.gcube.spatial-data</groupId>
<artifactId>ws-thredds</artifactId>
<version>0.2.0-SNAPSHOT</version>
<name>ws-thredds</name>
<description>prototype of WS integration with data-transfer for Thredds pubblication</description>
<properties>
<distroDirectory>${project.basedir}/distro</distroDirectory>
<svnBaseUrl>http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId}</svnBaseUrl>
</properties>
<scm>
<connection>scm:svn:${svnBaseUrl}/${project.artifactId}</connection>
<developerConnection>scm:svn:${svnBaseUrl}/${project.artifactId}</developerConnection>
<url>${svnBaseUrl}/${project.artifactId}</url>
</scm>
<dependencies>
<dependency>
<groupId>org.gcube.spatial.data</groupId>
<artifactId>sdi-library</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>data-transfer-library</artifactId>
<version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<!-- <dependency> -->
<!-- <groupId>org.gcube.data.transfer</groupId> -->
<!-- <artifactId>data-transfer-model</artifactId> -->
<!-- <version>1.2.4-4.11.0-163203</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.6.4</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>

0
requests.txt
Unescape Escape View File

162
src/main/java/org/gcube/usecases/ws/thredds/Commons.java
Unescape Escape View File

@ -0,0 +1,162 @@
package org.gcube.usecases.ws.thredds;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.text.SimpleDateFormat;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import org.apache.tika.io.IOUtils;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.client.AuthorizationFilter;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.faults.UnableToLockException;
import org.glassfish.jersey.client.ClientConfig;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Commons {
static final SimpleDateFormat DATE_FORMAT= new SimpleDateFormat("dd-MM-yy:HH:mm:SS");
public static void cleanupFolder(String toCleanPath, String destinationToken) {
String toRestoreToken=TokenSetter.getCurrentToken();
try{
log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath);
TokenSetter.setToken(destinationToken);
String hostname=getThreddsHost();
DataTransferClient client=getDTClient(hostname);
File toTransfer=File.createTempFile("clean", ".dt_temp");
toTransfer.createNewFile();
Destination dest=new Destination();
dest.setCreateSubfolders(true);
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
dest.setPersistenceId("thredds");
dest.setSubFolder("public/netcdf/"+toCleanPath);
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
client.localFile(toTransfer, dest);
log.info("Done");
}catch(Exception e) {
log.error("Unable to delete remote folder "+toCleanPath,e);
throw new RuntimeException("Unable to cleanup remote folder.");
}finally {
log.debug("Resetting original token {} ",toRestoreToken);
TokenSetter.set(toRestoreToken);
}
}
public static String getThreddsHost(){
SimpleQuery query =queryFor(GCoreEndpoint.class);
query.addCondition("$resource/Profile/ServiceClass/text() eq 'SDI'")
.addCondition("$resource/Profile/ServiceName/text() eq 'Thredds'");
// .setResult("$resource/Profile/AccessPoint");
DiscoveryClient<GCoreEndpoint> client = clientFor(GCoreEndpoint.class);
GCoreEndpoint endpoint= client.submit(query).get(0);
return endpoint.profile().endpoints().iterator().next().uri().getHost();
}
public static DataTransferClient getDTClient(String threddsHostName) throws UnreachableNodeException, ServiceNotFoundException {
log.debug("Getting DT Client for {} ",threddsHostName);
return DataTransferClient.getInstanceByEndpoint("http://"+threddsHostName+":80");
}
public static String readThreddsFile(String location) throws RemoteFileNotFoundException {
String urlString="http://"+getThreddsHost()+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+location;
log.info("Reading file at {} ",urlString);
try{
return getWebClient().target(urlString).request().get().readEntity(String.class);
}catch(Throwable t) {
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
}
}
public static void deleteThreddsFile(String location) throws RemoteFileNotFoundException {
String urlString="http://"+getThreddsHost()+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+location;
log.info("Reading file at {} ",urlString);
try{
getWebClient().target(urlString).request().delete();
}catch(Throwable t) {
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
}
}
public static final ThreddsInfo getThreddsInfo() {
String infoPath=getThreddsInfoPath();
log.info("Loading thredds info from {} ",infoPath);
WebTarget target=getWebClient().target(infoPath);
return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class);
}
private static Client getWebClient() {
return ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class));
}
private static String getThreddsInfoPath() {
return "https://"+getThreddsHost()+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";
}
public static void lockFolder(String folderPath,String processId) throws UnableToLockException {
PrintWriter writer=null;
File temp=null;
try{
log.info("Locking remote path {} to processId {} ",folderPath,processId);
DataTransferClient cl=getDTClient(getThreddsHost());
Destination dest=new Destination();
dest.setCreateSubfolders(false);
dest.setOnExistingFileName(DestinationClashPolicy.FAIL);
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
dest.setPersistenceId("thredds");
dest.setSubFolder(folderPath);
dest.setDestinationFileName(Constants.LOCK_FILE);
temp=File.createTempFile("tmp_lock", ".tmp");
writer=new PrintWriter(temp);
writer.write(processId);
writer.flush();
writer.close();
cl.localFile(temp, dest);
}catch(Throwable t) {
throw new UnableToLockException("Unable to lock "+folderPath,t);
}finally {
if(writer!=null) IOUtils.closeQuietly(writer);
if(temp!=null)try {
Files.deleteIfExists(temp.toPath());
}catch(IOException e) {
log.warn("Unable to delete temp file {} ",temp.getAbsolutePath(),e);
}
}
}
}

94
src/main/java/org/gcube/usecases/ws/thredds/Constants.java
Unescape Escape View File

@ -0,0 +1,94 @@
package org.gcube.usecases.ws.thredds;
import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.Map;
public class Constants {
public static final SimpleDateFormat DATE_FORMAT= new SimpleDateFormat("dd-MM-yy:HH:mm:SS");
public static final String LOCK_FILE="~WS-LOCK.lock";
public static final String GCUBE_TOKEN_HEADER="gcube-token";
public static final String LAST_UPDATE_TIME="WS-SYNCH.LAST_UPDATE";
public static final String THREDDS_PERSISTENCE="thredds";
public static final String THREDDS_DATA_TRANSFER_BASE_URL="data-transfer-service/gcube/service/REST/"+THREDDS_PERSISTENCE+"/";
public static final String SDI_THREDDS_BASE_URL="sdi-service/gcube/service/Thredds";
public static final String SIS_PLUGIN_ID="SIS/GEOTK";
public static class WorkspaceProperties{
// Folder
public static final String SYNCH_FILTER="WS-SYNCH.SYNCHRONIZATION-FILTER";
public static final String TARGET_TOKEN="WS-SYNCH.TARGET-TOKEN";
public static final String REMOTE_PATH="WS-SYNCH.REMOTE-PATH";
public static final String REMOTE_PERSISTENCE="WS-SYNCH.REMOTE-PERSISTENCE";
public static final String RELATED_CATALOG="WS-SYNCH.RELATED-CATALOG";
public static final String VALIDATE_METADATA="WS-SYNCH.VALIDATE-METADATA";
public static final String ROOT_FOLDER_ID="WS-SYNCH.ROOT-FOLDER-ID";
// Common
public static final String TBS="WS-SYNCH.TO-BE-SYNCHRONIZED";
public static final String SYNCHRONIZATION_STATUS="WS-SYNCH.SYNCH-STATUS";
public static final String LAST_UPDATE_TIME="WS-SYNCH.LAST-UPDATE-TIME";
public static final String LAST_UPDATE_STATUS="WS-SYNCH.LAST-UPDATE-STATUS";
// ITEM
public static final String METADATA_UUID="WS-SYNCH.METADATA-UUID";
}
public static class Configuration{
public static final String SCANNER_POOL_MAX_SIZE="scanner.pool.maxSize";
public static final String SCANNER_POOL_CORE_SIZE="scanner.pool.coreSize";
public static final String SCANNER_POOL_IDLE_MS="scanner.pool.idle.ms";
public static final String TRANSFERS_POOL_MAX_SIZE="transfers.pool.maxSize";
public static final String TRANSFERS_POOL_CORE_SIZE="transfers.pool.coreSize";
public static final String TRANSFERS_POOL_IDLE_MS="transfers.pool.idle.ms";
}
public static final Map<String,String> cleanedItemPropertiesMap=new HashMap<String,String>();
public static final Map<String,String> cleanedFolderPropertiesMap=new HashMap<String,String>();
public static final Map<String,String> defaultConfigurationMap=new HashMap<String,String>();
static {
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.TBS, null);
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, null);
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, null);
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, null);
cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.METADATA_UUID, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.SYNCH_FILTER, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.REMOTE_PATH, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.REMOTE_PERSISTENCE, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.TARGET_TOKEN, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.TBS, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.RELATED_CATALOG, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.VALIDATE_METADATA, null);
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.ROOT_FOLDER_ID, null);
defaultConfigurationMap.put(Configuration.SCANNER_POOL_CORE_SIZE, "1");
defaultConfigurationMap.put(Configuration.SCANNER_POOL_MAX_SIZE, "10");
defaultConfigurationMap.put(Configuration.SCANNER_POOL_IDLE_MS, "30000");
defaultConfigurationMap.put(Configuration.TRANSFERS_POOL_CORE_SIZE, "1");
defaultConfigurationMap.put(Configuration.TRANSFERS_POOL_MAX_SIZE, "10");
defaultConfigurationMap.put(Configuration.TRANSFERS_POOL_IDLE_MS, "30000");
}
}

40
src/main/java/org/gcube/usecases/ws/thredds/FolderConfiguration.java
Unescape Escape View File

@ -0,0 +1,40 @@
package org.gcube.usecases.ws.thredds;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@RequiredArgsConstructor
@ToString
public class FolderConfiguration {
public FolderConfiguration(FolderConfiguration other) {
this.providedMetadata=other.providedMetadata;
this.includeSubfolders=other.includeSubfolders;
this.publishingUserToken=other.publishingUserToken;
this.folderId=other.folderId;
this.catalogName=other.catalogName;
this.metadataFolderId=other.metadataFolderId;
}
private boolean providedMetadata=false;
private boolean includeSubfolders=true;
@NonNull
private String publishingUserToken;
@NonNull
private String folderId;
@NonNull
private String catalogName;
private String metadataFolderId=null;
}

31
src/main/java/org/gcube/usecases/ws/thredds/FolderReport.java
Unescape Escape View File

@ -0,0 +1,31 @@
package org.gcube.usecases.ws.thredds;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@RequiredArgsConstructor
public class FolderReport {
@NonNull
private FolderConfiguration config;
private Set<String> transferredFiles=new HashSet<>();
/* map file->uuid*/
private Map<String,String> generatedMetadata=new HashMap<>();
private Set<String> publishedMetadata=new HashSet<>();
}

44
src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java
Unescape Escape View File

@ -0,0 +1,44 @@
package org.gcube.usecases.ws.thredds;
import java.util.Properties;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LocalConfiguration {
private static LocalConfiguration instance=null;
@Synchronized
private static final LocalConfiguration get() {
if(instance==null)
instance=new LocalConfiguration();
return instance;
}
public static String getProperty(String property) {
try{
return (String) get().props.getOrDefault(property, Constants.defaultConfigurationMap.get(property));
}catch(Throwable t) {
log.warn("Unable to get configuration property "+property,t);
return Constants.defaultConfigurationMap.get(property);
}
}
//***************** INSTANCE
Properties props;
public LocalConfiguration() {
props=new Properties();
try{
props.load(this.getClass().getResourceAsStream("configuration.properties"));
}catch(Exception e) {
log.warn("********************** UNABLE TO LOAD PROPERTIES **********************",e);
log.debug("Reverting to defaults : "+Constants.defaultConfigurationMap);
}
}
}

57
src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java
Unescape Escape View File

@ -0,0 +1,57 @@
package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.commons.io.IOUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class NetUtils {
public static final File toFile(InputStream is) {
FileOutputStream outputStream =null;
try {
File toReturn=File.createTempFile("tempFile", ".tmp");
outputStream=new FileOutputStream(toReturn);
int read = 0;
byte[] bytes = new byte[1024];
while ((read = is.read(bytes)) != -1) {
outputStream.write(bytes, 0, read);
}
return toReturn;
}catch(Throwable t) {
throw new RuntimeException(t);
}finally {
if(outputStream!=null) {
IOUtils.closeQuietly(outputStream);
}
}
}
public static String resolveRedirects(String url) throws IOException{
log.debug("Resolving redirect for url {} ",url);
URL urlObj=new URL(url);
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
int status=connection.getResponseCode();
if(status>=300&&status<400){
String newUrl=connection.getHeaderField("Location");
log.debug("Following redirect from {} to {} ",url,newUrl);
return resolveRedirects(newUrl);
}else return url;
}
public static File download(String url) throws MalformedURLException, IOException {
return toFile(new URL(resolveRedirects(url)).openStream());
}
}

283
src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java
Unescape Escape View File

@ -0,0 +1,283 @@
package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WrongItemTypeException;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
import org.gcube.usecases.ws.thredds.engine.PublishRequest;
import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
import org.gcube.usecases.ws.thredds.engine.PublishRequest.PublishItem;
import org.gcube.usecases.ws.thredds.engine.TransferRequestServer;
import org.gcube.usecases.ws.thredds.engine.TransferRequestServer.Report;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class PublishFolders {
private static String WS_USER="fabio.sinibaldi";
public static void main(String[] args) {
TokenSetter.set("/d4science.research-infrastructures.eu");
HashSet<FolderConfiguration> configs=new HashSet<>();
// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS
// String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files
String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST
// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET
String publishingUserToken="5741e3e4-dbde-46fa-828d-88da609e0517-98187548"; //fabio @NextNext
FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE");
// folderConfig.setProvidedMetadata(true);
// folderConfig.setMetadataFolderId(metadataFolderID);
configs.add(folderConfig);
TransferRequestServer server=new TransferRequestServer();
for(FolderConfiguration entry:configs){
try{
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace();
// FolderReport report=new FolderReport(entry);
log.info("Managing {} ",entry);
WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId());
handleFolder(ws,entry,server,folder);
}catch(WorkspaceException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(HomeNotFoundException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(InternalErrorException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(UserNotFoundException e){
System.err.println("WORKSPACE EXC ");
e.printStackTrace(System.err);
}catch(Exception e){
System.err.println("UNEXPECTED EXC");
e.printStackTrace(System.err);
}
}
System.out.println("Waiting for service.. ");
server.waitCompletion();
Report report=server.getReport();
File reportFile =report.toFile(folderConfig);
System.out.println("Report at "+reportFile.getAbsolutePath());
}
/**
* For *.nc | *.ncml
* if relatedMetadataFolder contains <filename>.xml use meta
* else ask DT to generate it
*
*
* @param wsFolder
* @param config
* @param server
* @return
* @throws WorkspaceException
* @throws InternalErrorException
* @throws ItemNotFoundException
* @throws HomeNotFoundException
* @throws UserNotFoundException
*/
public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
log.info("Handling folder {} ",folder.getPath());
cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName());
log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath());
publishFolder(ws,config,server,folder);
}
public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{
//Access folder
List<WorkspaceItem> folderItems=folder.getChildren();
if(config.isIncludeSubfolders()) {
log.info("Going through subfolders first.....");
for(WorkspaceItem item:folderItems) {
try {
if(item.isFolder()) {
FolderConfiguration subConfig=new FolderConfiguration(config);
subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName());
publishFolder(ws,subConfig,server,(WorkspaceFolder) item);
log.debug("Subfolder {} successfully analyzed ",item.getPath());
}
}catch(Exception e) {
try{
log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e);
}
}
}
}
log.debug("Checking for ncml files .... ");
for(WorkspaceItem item : folderItems) {
try {
if(!item.isFolder()) {
String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
if(prefix.equals(".ncml")) {
PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken());
if(config.isProvidedMetadata()) {
String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
if (meta!=null) req.setMetadata(meta);
}
// TODO NB Check for queue
server.put(req);
}
}
}catch(Exception e) {
try{
log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e);
}
}
}
log.debug("Checking nc files.. ");
for(WorkspaceItem item:folder.getChildren()){
try {
if(!item.isFolder()) {
String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length());
if(prefix.equals(".nc")){
// NC
PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken());
if(config.isProvidedMetadata()) {
String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml";
File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId());
if (meta!=null) req.setMetadata(meta);
}
server.put(req);
}
}
}catch(Exception e) {
try{
log.warn("Unabel to check item {} ",item.getPath(),e);
}catch(InternalErrorException e1) {
log.warn("Unabel to check item and to get Path {} ",item,e);
}
}
}
folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+""));
log.debug("Creating requests... ");
}
private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException {
try{
WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId);
if(found==null) throw new ItemNotFoundException("Found item was null");
return NetUtils.toFile(((ExternalFile)found).getData());
}catch(ItemNotFoundException e) {
return null;
}
}
// Remotely deletes folders which has history operations : RENAMING, REMOVAL
private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException {
List<AccountingEntry> history=folder.getAccounting();
long lastUpdateTimeMillis=getLastUpdateTime(folder);
Date lastUpdate=new Date(lastUpdateTimeMillis);
log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate));
//look into history
boolean toDeleteCurrentFolder=false;
for(AccountingEntry entry: history) {
Date eventTime=entry.getDate().getTime();
switch(entry.getEntryType()) {
case REMOVAL:
case RENAMING:{
log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
if(eventTime.after(lastUpdate)) {
log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime));
toDeleteCurrentFolder=true;
}
}
}
if(toDeleteCurrentFolder) break;
}
//Delete Folder or scan children
if(toDeleteCurrentFolder) {
log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath);
try{
Commons.cleanupFolder(remoteFolderPath,targetToken);
}catch(Throwable t) {
log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t);
if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish
throw t;
}
}else {
log.info("Folder is not to be cleaned up. Checking children..");
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName());
}
}
private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException {
try{
return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME));
}catch(Throwable e) {
log.debug("Unable to get last update time for {}. Considering 0..",item.getPath());
return 0l;
}
}
}

57
src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java
Unescape Escape View File

@ -0,0 +1,57 @@
package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.util.Set;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
import org.gcube.usecases.ws.thredds.engine.impl.SynchEngineImpl;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo;
import org.gcube.usecases.ws.thredds.model.gui.CatalogBean;
public interface SyncEngine {
public static SyncEngine get() {
return SynchEngineImpl.get();
}
public SyncFolderDescriptor check(String folderId, boolean recursively) throws WorkspaceInteractionException, InternalException;
public void registerCallBack(String folderId,SyncOperationCallBack callback) throws ProcessNotFoundException;
public ProcessDescriptor doSync(String folderId) throws WorkspaceInteractionException, InternalException;
public void stopSynch(String folderId) throws ProcessNotFoundException;
public void setSynchronizedFolder(SynchFolderConfiguration config,String folderId) throws WorkspaceInteractionException, InternalException;
public void unsetSynchronizedFolder(String folderId,boolean deleteRemoteContent) throws WorkspaceInteractionException, InternalException;
public SynchronizedElementInfo getInfo(String elementId);
public void updateCatalogFile(String folderId, File toUpdate) throws InternalException;
public void forceUnlock(String folderId)throws InternalException, WorkspaceInteractionException;
public void shutDown();
public ProcessDescriptor getProcessDescriptorByFolderId(String folderId)throws ProcessNotFoundException;
public ProcessStatus getProcessStatusByFolderId(String folderId)throws ProcessNotFoundException;
public void setRequestLogger(String path);
public boolean isRequestLoggerEnabled();
public String getRequestLoggerPath();
public Set<CatalogBean> getAvailableCatalogsByToken(String token) throws InternalException;
}

55
src/main/java/org/gcube/usecases/ws/thredds/TokenSetter.java
Unescape Escape View File

@ -0,0 +1,55 @@
package org.gcube.usecases.ws.thredds;
import static org.gcube.common.authorization.client.Constants.authorizationService;
import java.util.Properties;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TokenSetter {
private static Properties props=null;
static{
}
public static synchronized void set(String scope){
try{
if(props==null) {
props=new Properties();
try {
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
} catch (Exception e) {
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
}
}
if(!props.containsKey(scope)) throw new Exception("No token found for scope : "+scope);
SecurityTokenProvider.instance.set(props.getProperty(scope));
}catch(Throwable e){
log.trace("Unable to set token for scope "+scope,e);
}
ScopeProvider.instance.set(scope);
}
public static void setToken(String token){
try{
AuthorizationEntry entry = authorizationService().get(token);
ScopeProvider.instance.set(entry.getContext());
SecurityTokenProvider.instance.set(token);
}catch(Throwable t) {
throw new RuntimeException("Unable to set token "+token,t);
}
}
public static String getCurrentToken() {
return SecurityTokenProvider.instance.get();
}
}

22
src/main/java/org/gcube/usecases/ws/thredds/engine/PublishReport.java
Unescape Escape View File

@ -0,0 +1,22 @@
package org.gcube.usecases.ws.thredds.engine;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class PublishReport {
boolean isError=false;
private String sourceId;
private String sourceName;
private TransferResult transferResult;
private MetadataReport metadataReport;
}

64
src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java
Unescape Escape View File

@ -0,0 +1,64 @@
package org.gcube.usecases.ws.thredds.engine;
import java.io.File;
import java.util.HashSet;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NonNull;
import lombok.ToString;
@Data
public class PublishRequest {
@Getter
@AllArgsConstructor
@ToString
public static class PublishItem{
String url;
String name;
String id;
public PublishItem(WorkspaceItem item) throws InternalErrorException {
url=item.getPublicLink(false);
name=item.getName();
id=item.getId();
}
}
public static enum Mode{
NCML,NC
}
@NonNull
private PublishItem source;
@NonNull
private Mode mode;
@NonNull
private String catalog;
@NonNull
private String publishToken;
private Integer queueCount=0;
private String queueId;
private File metadata=null;
private HashSet<String> toGatherReportsId=null;
public boolean isQueue() {
return queueCount>0;
}
public boolean isGenerateMeta() {
return metadata==null;
}
}

217
src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java
Unescape Escape View File

@ -0,0 +1,217 @@
package org.gcube.usecases.ws.thredds.engine;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
import org.gcube.usecases.ws.thredds.Commons;
import org.gcube.usecases.ws.thredds.NetUtils;
import org.gcube.usecases.ws.thredds.TokenSetter;
import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class PublishThread implements Runnable {
@NonNull
private PublishRequest request;
@NonNull
private ConcurrentHashMap<String,PublishReport> reports;
private PublishReport publishReport;
// private Map<String,Report> reports;
//
@Override
public void run() {
log.info("Request is {}",request);
log.debug("Switching from {} to {}",SecurityTokenProvider.instance.get(),request.getPublishToken());
TokenSetter.setToken(request.getPublishToken());
log.debug("Current scope is :{}, token is {} ",ScopeUtils.getCurrentScope(),SecurityTokenProvider.instance.get());
Destination dest=new Destination();
dest.setPersistenceId("thredds");
dest.setSubFolder("public/netcdf/"+request.getCatalog());
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
dest.setCreateSubfolders(true);
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
String threddsHostName;
try {
threddsHostName = Commons.getThreddsHost();
DataTransferClient client=Commons.getDTClient(threddsHostName);
File toPublishSource=null;
if(request.getMode().equals(Mode.NCML)) {
if(request.isQueue()){
log.debug("Waiting for queue {}, expected Count {} ",request.getQueueId(),request.getQueueCount());
waitFor(request.getQueueId(), request.getQueueCount());
log.debug("Loading netcdfFile ..");
File ncmlFile=NetUtils.download(request.getSource().getUrl());
String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath()));
for(String reportId:request.getToGatherReportsId()) {
PublishReport report=getReport(reportId);
//file://home/gcube/julien.barde/Workspace/DataMiner/Output_Data_Sets/Ichthyop2013.nc
String toSetUrl="file:/"+report.getTransferResult().getRemotePath();
toUpdateSource=toUpdateSource.replaceAll(reportId, toSetUrl);
}
toPublishSource=File.createTempFile("nc_", ".ncml");
PrintWriter out = new PrintWriter(toPublishSource);
out.write(toUpdateSource);
out.flush();
}
}
TransferResult result;
// TODO NB Test run without metadata publication
// result=client.httpSource(request.getSource().getUrl(), dest);
// publishReport=new PublishReport(false,request.getSource().getId(),result,null);
if(!request.isGenerateMeta()) {
log.debug("Transfering before publishing meta..");
result = toPublishSource==null?client.httpSource(request.getSource().getUrl(), dest):
client.localFile(toPublishSource, dest);
//NB DECOMMENT THIS!!!!!!
// Metadata meta=SDIAbstractPlugin.metadata().build();
//
// log.debug("Publishing metadata.. ");
//
// MetadataPublishOptions opts=new MetadataPublishOptions(new TemplateInvocationBuilder().threddsOnlineResources(threddsHostName, request.getSource().getName(), request.getCatalog()).get());
// opts.setGeonetworkCategory("Datasets");
// MetadataReport report=meta.pushMetadata(request.getMetadata(), opts);
// publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,report);
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,new MetadataReport());
}else {
log.debug("Metadata not provided.. ");
if(request.isQueue()&&request.getMode().equals(Mode.NC)) {
log.debug("Dataset file is linked in ncml, skipping metadata generation");
result=client.httpSource(request.getSource().getUrl(), dest);
}else
result=client.httpSource(request.getSource().getUrl(), dest,new PluginInvocation("SIS/GEOTK"));
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,null);
}
} catch (UnreachableNodeException | ServiceNotFoundException e) {
log.error("Unable to find Thredds. Publish scope is {} ",ScopeUtils.getCurrentScope(),e);
} catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException
| InvalidDestinationException | DestinationNotSetException e) {
log.error("Unable to transfer file, ",e);
} catch (IOException e) {
log.error("Unable to read/ write file. ",e);
}
onCompletion();
}
private void onCompletion() {
if(publishReport==null) publishReport=new PublishReport(true, request.getSource().getId(),request.getSource().getName(), null, null);
publishReport(publishReport);
if(request.getMode().equals(Mode.NC)&&(request.isQueue())) {
alert(request.getQueueId(),request.getQueueCount());
}
}
private PublishReport getReport(String reportId) {
return reports.get(reportId);
}
private void publishReport(PublishReport report) {
reports.put(report.getSourceId(), report);
}
// static
private static ConcurrentHashMap<String,Semaphore> semaphores=new ConcurrentHashMap<>();
private static void waitFor(String queueId,Integer expected) {
try {
log.debug("Waiting for queue {}. Expected Count is {} ",queueId,expected);
semaphores.getOrDefault(queueId, new Semaphore(expected*-1)).acquire();
} catch (InterruptedException e) {
log.debug("Queue {} is completed.");
}
}
private static void alert(String queueId, Integer expected) {
log.debug("Alerting queue {}. Expected count is {} ",queueId,expected);
Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1));
sem.release();
log.debug("Queue {} alerted. Remaining : {} out of {} ",queueId,sem.availablePermits(),expected);
}
}

102
src/main/java/org/gcube/usecases/ws/thredds/engine/TransferRequestServer.java
Unescape Escape View File

@ -0,0 +1,102 @@
package org.gcube.usecases.ws.thredds.engine;
import java.io.File;
import java.io.PrintWriter;
import java.util.Map.Entry;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.IOUtils;
import org.gcube.usecases.ws.thredds.FolderConfiguration;
import lombok.Data;
import lombok.Synchronized;
public class TransferRequestServer {
@Data
public static class Report{
private AtomicLong requestCount=new AtomicLong(0l);
private AtomicLong requestServed=new AtomicLong(0l);
private ConcurrentHashMap<String,PublishReport> reports=new ConcurrentHashMap<>();
public File toFile(FolderConfiguration configuration) {
return toFile(configuration,this);
}
private static final File toFile(FolderConfiguration config,Report report) {
PrintWriter writer =null;
try {
File toReturn=File.createTempFile("tempFile", ".tmp");
writer=new PrintWriter(toReturn);
writer.println("REPORT FOR WS-SYNCH");
writer.println("Configuratiion was : "+config);
writer.println("Submitted runs : "+report.getRequestCount());
writer.println("Item reports : ");
for(Entry<String,PublishReport> entry: report.getReports().entrySet()) {
PublishReport rep=entry.getValue();
writer.println("*********************************************************");
if(rep.isError()) writer.println("OPERATION IS FAILED");
writer.println("ITEM ID : "+rep.getSourceId());
writer.println("ITEM NAME : "+rep.getSourceName());
if(rep.getTransferResult()!=null)writer.println("Transfer report : "+rep.getTransferResult());
if(rep.getMetadataReport()!=null)writer.println("Metadata report : "+rep.getMetadataReport());
}
return toReturn;
}catch(Throwable t) {
throw new RuntimeException(t);
}finally {
if(writer!=null) {
IOUtils.closeQuietly(writer);
}
}
}
}
private Report report=new Report();
private ExecutorService service=null;
public TransferRequestServer() {
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(
100);
service= new ThreadPoolExecutor(1, 10, 30,
TimeUnit.SECONDS, linkedBlockingDeque,
new ThreadPoolExecutor.CallerRunsPolicy());
}
public void put(PublishRequest request){
System.out.println("Submitting transfer "+getReport().requestCount.incrementAndGet());
service.execute(new PublishThread(request, getReport().getReports()));
// service.execute(new RequestThread(baseUrl,filename,this,publishScope,toPublishMeta));
}
@Synchronized
public Report getReport(){
return report;
}
public void waitCompletion() {
boolean running=true;
service.shutdown();
while(running){
System.out.println("******************* WAITING FOR TERMINATION ***************** ");
try{
running=!service.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
}catch(InterruptedException e){
running=!service.isTerminated();
}
}
System.out.println("Service is completed : "+service.isTerminated());
}
}

312
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java
Unescape Escape View File

@ -0,0 +1,312 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status;
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferToThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.CompletionCallback;
import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Process {
private boolean submittedRequests=false;
private ProcessDescriptor descriptor;
private ProcessStatus status;
private String processId=UUID.randomUUID().toString();
private Queue<StepReport> queuedReports=new LinkedList<>();
// private String folderId;
private WorkspaceFolderManager manager;
private Set<SyncOperationCallBack> toInvokeCallbacks=ConcurrentHashMap.newKeySet();
private CompletionCallback callback=null;
public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException {
log.debug("Created Process with id {} ",processId);
// this.folderId=folderId;
manager=new WorkspaceFolderManager(folderId);
manager.lock(processId);
SynchFolderConfiguration folderConfig=manager.getSynchConfiguration();
try {
descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().getPath(),System.currentTimeMillis(),processId,folderConfig);
}catch(Exception e) {
throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e);
}
this.callback=callback;
status=new ProcessStatus();
}
public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalErrorException, InternalException {
WorkspaceUtils.resetStatus(manager.getTheFolder());
status.setCurrentMessage("Analyzing folder..");
generateRequests(this,service, manager.getTheFolder());
submittedRequests=true;
if(status.getQueuedTransfers().get()>0) {
status.setCurrentMessage("Waiting for requests ["+status.getQueuedTransfers().get()+"] to be served.");
status.setStatus(Status.ONGOING);
while(!queuedReports.isEmpty()) {
onStep(queuedReports.remove());
}
}else {
status.setCurrentMessage("Folder is up to date.");
status.setStatus(Status.COMPLETED);
callback.onProcessCompleted(this);
invokeCallbacks();
}
}
public void addCallBack(SyncOperationCallBack toAddCallback) {
toInvokeCallbacks.add(toAddCallback);
log.debug("Added callback for process {}. Current callback size is {}",processId,toInvokeCallbacks.size());
}
public ProcessDescriptor getDescriptor() {
return descriptor;
}
// signals from serving threads
public void onStep(StepReport report) {
if(!submittedRequests) {
queuedReports.add(report);
}else {
// serve
updateStatus(report);
if(isCompleted()) {
try {
manager.setLastUpdateTime();
}catch(Throwable t) {
log.error("Unable to update last update time.",t);
}
if(status.getStatus().equals(Status.WARNINGS))
status.setCurrentMessage("Process completed with errors. Please check logs or retry.");
else status.setCurrentMessage("Synchronization complete.");
status.setStatus(Status.COMPLETED);
callback.onProcessCompleted(this);
}
invokeCallbacks();
}
}
private void invokeCallbacks() {
for(SyncOperationCallBack callback:toInvokeCallbacks) {
try {
callback.onStep((ProcessStatus)status.clone(), (ProcessDescriptor)descriptor.clone());
}catch(Throwable t) {
log.warn("Unable to invoke callback {}.",callback,t);
}
}
}
private boolean isCompleted() {
return (status.getErrorCount().get()+status.getServedTransfers().get()>=status.getQueuedTransfers().get());
}
@Synchronized
public void updateStatus(StepReport report) {
log.debug("Logging report {} ",report);
switch(report.getStatus()) {
case CANCELLED :
case ERROR:{
status.getErrorCount().incrementAndGet();
if(!status.getStatus().equals(Status.STOPPED))
status.setStatus(Status.WARNINGS);
break;
}
default : {
status.getServedTransfers().incrementAndGet();
break;
}
}
status.getLogBuilder().append(
String.format("%s - item [%s] %s: %s \n", Constants.DATE_FORMAT.format(new Date(report.getCompletionTime())),
report.getElementName(),report.getStatus()+"",report.getMessage()));
}
public ProcessStatus getStatus() {
return status;
}
public void cancel() {
if(status.getQueuedTransfers().get()>1) {
status.setStatus(Status.STOPPED);
status.setCurrentMessage("Process Stopped. Waiting for remaining requests to cancel..");
}else {
status.setStatus(Status.COMPLETED);
status.setCurrentMessage("Process cancelled before it started.");
}
invokeCallbacks();
callback.onProcessCompleted(this);
}
public void cleanup() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
manager.unlock(processId);
}
@Override
protected void finalize() throws Throwable {
try {
cleanup();
}catch(Throwable t) {
log.warn("Exception while trying to cleanup {} ",this);
}
}
private static final void generateRequests(Process ownerProcess,ExecutorService service,WorkspaceFolder toScanFolder ) throws InternalErrorException, InternalException{
String folderPath=toScanFolder.getPath();
log.info("Generating requests for folder {}",folderPath);
log.debug("Process is {} ",ownerProcess.getDescriptor());
Set<String> handledWorkspaceItemEntries=new HashSet<String>();
SynchFolderConfiguration config=ownerProcess.getDescriptor().getSynchConfiguration();
Set<String> remoteChildrenNames;
Set<String> localChildrenNames=new HashSet<>();
List<WorkspaceItem> localFolderChildren=toScanFolder.getChildren();
for(WorkspaceItem item:localFolderChildren) {
localChildrenNames.add(item.getName());
}
String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken());
RemoteFileDescriptor folderDesc=null;
try{
folderDesc=folderController.getFileDescriptor();
}catch(RemoteFileNotFoundException e) {
log.debug("RemoteFolder {} doesn't exists. Creating it.. ",relativePath);
folderController.createEmptyFolder(null);
folderDesc=folderController.getFileDescriptor();
}
remoteChildrenNames=new HashSet<>(folderDesc.getChildren());
//*********************** HANDLING ACCOUNTING ENTRIES
Set<String> handledAccountingEntries=WorkspaceUtils.scanAccountingForStatus( toScanFolder, config, localChildrenNames, remoteChildrenNames, folderController, ownerProcess, service);
//SCAN FOLDER CONTENT
log.debug("Checking content of {} ",folderPath);
for(WorkspaceItem item:localFolderChildren) {
if(item.isFolder()) {
// RECURSIVE ON SUB FOLDERS
generateRequests(ownerProcess,service,(WorkspaceFolder) item);
}else {
Map<String,String> props=item.getProperties().getProperties();
String itemId=item.getId();
String itemName=item.getName();
// REQUESTS ARE EVALUATED ON PROPERTIES (SET BY PREVIOUS SCAN)
if(props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null)) {
try {
SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS));
log.trace("Found TBS item {}, name {}, status : ",item.getId(),item.getName(),status);
SynchronizationRequest request=null;
switch(status) {
case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder,item);
break;
case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item, toScanFolder, null);
break;
}
if(request!=null) {
// KEEP TRACK OF HANDLED ITEMS & LAUNCH
service.execute(new SynchronizationThread(request));
log.debug("Submitted request number {} ",ownerProcess.status.getQueuedTransfers().incrementAndGet());
}else log.debug("Item is up to date");
handledWorkspaceItemEntries.add(itemName);
}catch(Throwable t) {
log.error("Unable to submit request for {} ID {} ",itemName,itemId,t);
}
}
}
}
// check items to be imported
try {
Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, handledAccountingEntries, handledWorkspaceItemEntries, toScanFolder, folderController, config, ownerProcess, service);
log.debug("Checking if remote location contains folders to be imported...");
for(String item:toImportItems) {
if(folderController.getFileDescriptor(item).isDirectory()) {
log.info("Creating folder {} under {} ",item,folderPath);
try{
WorkspaceFolder folder=toScanFolder.createFolder(item, "Imported from thredds");
WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId());
generateRequests(ownerProcess, service, folder);
}catch(Throwable t) {
log.error("Unable to import folder {} into {} ",item,folderPath);
}
}
}
}catch(InternalException e) {
log.error("Unable to check remote content with config {} ",config,e);
}
log.info("All requests for {} synchronization have been submitted [count {} ]. ",folderPath,ownerProcess.status.getQueuedTransfers().get());
}
}

23
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessDescriptor.java
Unescape Escape View File

@ -0,0 +1,23 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class ProcessDescriptor implements Cloneable{
private String folderId;
private String folderPath;
private long launchTime;
private String processId;
private SynchFolderConfiguration synchConfiguration;
@Override
protected Object clone() throws CloneNotSupportedException {
return super.clone();
}
}

50
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessStatus.java
Unescape Escape View File

@ -0,0 +1,50 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.util.concurrent.atomic.AtomicLong;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class ProcessStatus implements Cloneable{
public static enum Status{
INITIALIZING, // initial status
ONGOING, // synch in progress
WARNINGS, // errors occurred, still WORKING
STOPPED, // STOP received, waiting for request to finish
COMPLETED // FINISHED PROCESS
}
private AtomicLong queuedTransfers=new AtomicLong(0);
private AtomicLong servedTransfers=new AtomicLong(0);
private AtomicLong errorCount=new AtomicLong(0);
private Status status=Status.INITIALIZING;
private StringBuilder logBuilder=new StringBuilder();
private String currentMessage="Waiting to start..";
@Override
protected Object clone() throws CloneNotSupportedException {
return super.clone();
}
@Override
public String toString() {
return "ProcessStatus [queuedTransfers=" + queuedTransfers + ", servedTransfers=" + servedTransfers
+ ", errorCount=" + errorCount + ", status=" + status + ", currentMessage=" + currentMessage + "]";
}
public float getPercent() {
switch(status) {
case INITIALIZING : return 0;
case COMPLETED : return 1;
default : return queuedTransfers.get()==0?0:((float)(servedTransfers.get()+errorCount.get()))/queuedTransfers.get();
}
}
}

267
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java
Unescape Escape View File

@ -0,0 +1,267 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.io.File;
import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.gcube.data.transfer.model.plugins.thredds.DataSetScan;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.LocalConfiguration;
import org.gcube.usecases.ws.thredds.SyncEngine;
import org.gcube.usecases.ws.thredds.engine.impl.threads.ProcessInitializationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.RequestLogger;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceFolderNotRootException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.CompletionCallback;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo;
import org.gcube.usecases.ws.thredds.model.gui.CatalogBean;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SynchEngineImpl implements SyncEngine{
private static SynchEngineImpl instance=null;
public static synchronized SyncEngine get() {
if(instance==null) {
instance=new SynchEngineImpl();
}
return instance;
}
private SynchEngineImpl() {
localProcesses=new ConcurrentHashMap<>();
int scannerMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_MAX_SIZE));
int scannerCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_CORE_SIZE));
int scannerIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_IDLE_MS));
int transfersMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_MAX_SIZE));
int transfersCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_CORE_SIZE));
int transfersIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_IDLE_MS));
initializationExecutor= new ThreadPoolExecutor(scannerCoreSize, scannerMaxSize, scannerIdleMs,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
synchronizationExecutor=new ThreadPoolExecutor(transfersCoreSize, transfersMaxSize, transfersIdleMs,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
}
private String requestLoggerPath=null;
@Override
public void setRequestLogger(String path) {
requestLoggerPath=path;
}
@Override
public boolean isRequestLoggerEnabled() {
return requestLoggerPath!=null;
}
@Override
public String getRequestLoggerPath() {
return requestLoggerPath;
}
//folder ID -> Process
private ConcurrentHashMap<String, Process> localProcesses;
private ExecutorService initializationExecutor=null;
private ExecutorService synchronizationExecutor=null;
private final CompletionCallback completionCallback=new CompletionCallback() {
@Override
public void onProcessCompleted(Process completedProcess) {
try {
ProcessDescriptor descriptor=completedProcess.getDescriptor();
log.info("Process {} is completed. Going to cleanup.. ",descriptor);
localProcesses.remove(descriptor.getFolderId());
completedProcess.cleanup();
}catch(Throwable t) {
log.warn("Unable to cleanup {} ",completedProcess,t);
}
}
};
@Override
public SyncFolderDescriptor check(String folderId, boolean recursively) throws WorkspaceInteractionException, InternalException {
WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId);
return manager.check(recursively);
}
@Override
public void registerCallBack(String folderId, SyncOperationCallBack callback) throws ProcessNotFoundException {
if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under local processes");
localProcesses.get(folderId).addCallBack(callback);
}
@Override
public ProcessDescriptor doSync(String folderId) throws WorkspaceInteractionException, InternalException {
if(localProcesses.containsKey(folderId))
return localProcesses.get(folderId).getDescriptor();
else {
WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId);
if (!manager.isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not configured for synchronization.");
if(manager.isLocked()) throw new WorkspaceLockedException("Folder "+folderId+"is locked by an external process.");
if(!manager.isRoot()) throw new WorkspaceFolderNotRootException("Unable to launch synch operation. Folder "+folderId+" is not root configuration");
Process toLaunch=new Process(folderId,completionCallback);
localProcesses.put(folderId, toLaunch);
initializationExecutor.submit(new ProcessInitializationThread(toLaunch,synchronizationExecutor));
return toLaunch.getDescriptor();
}
}
@Override
public void stopSynch(String folderId) throws ProcessNotFoundException {
if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under local processes");
localProcesses.get(folderId).cancel();
}
@Override
public void setSynchronizedFolder(SynchFolderConfiguration config,String folderId) throws WorkspaceInteractionException, InternalException {
// Check config
if(config==null) throw new InternalException("Passed config is null : "+config);
String remotePath=config.getRemotePath();
if(remotePath==null||remotePath.isEmpty()||remotePath.startsWith("/"))
throw new InternalException("Invalid remote path "+remotePath+".");
new WorkspaceFolderManager(folderId).configure(config);
}
@Override
public void unsetSynchronizedFolder(String folderId,boolean deleteRemoteContent) throws WorkspaceInteractionException, InternalException {
new WorkspaceFolderManager(folderId).dismiss(deleteRemoteContent);
}
@Override
public SynchronizedElementInfo getInfo(String elementId) {
return WorkspaceFolderManager.getInfo(elementId);
}
@Override
public void updateCatalogFile(String folderId, File toUpdate) throws InternalException {
File previousCatalogFile=null;
try {
WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId);
previousCatalogFile=manager.loadCatalogFile();
String lockId=UUID.randomUUID().toString();
manager.lock(lockId);
manager.updateCatalogFile(toUpdate);
manager.unlock(lockId);
}catch(Throwable t) {
log.warn("Unable to update catalogFile for {}. Trying to restore previous one..",folderId,t);
throw new InternalException("Unable to restore previous catalog.",t);
//TODO try to restore previous catalog
}
}
@Override
public void shutDown() {
log.trace("Cancelling processes...");
for(Entry<String,Process> entry:localProcesses.entrySet())
entry.getValue().cancel();
log.trace("Shutting down services... ");
initializationExecutor.shutdown();
synchronizationExecutor.shutdown();
do {
log.trace("Waiting for services to terminate..");
try {Thread.sleep(1000l);
} catch (InterruptedException e) {}
}while(!initializationExecutor.isTerminated()||!synchronizationExecutor.isTerminated());
RequestLogger.get().close();
log.trace("Terminated.");
}
@Override
public void forceUnlock(String folderId) throws InternalException, WorkspaceInteractionException {
log.warn("Forcing unlock of {} ",folderId);
new WorkspaceFolderManager(folderId).forceUnlock();
}
@Override
public ProcessDescriptor getProcessDescriptorByFolderId(String folderId) throws ProcessNotFoundException {
if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under processes or process is not in this host");
return localProcesses.get(folderId).getDescriptor();
}
@Override
public ProcessStatus getProcessStatusByFolderId(String folderId) throws ProcessNotFoundException {
if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under processes or process is not in this host");
return localProcesses.get(folderId).getStatus();
}
@Override
public Set<CatalogBean> getAvailableCatalogsByToken(String token) throws InternalException {
ThreddsController controller=new ThreddsController("",token);
ThreddsInfo info=controller.getThreddsInfo();
Set<CatalogBean> toReturn=asCatalogBeanSet(info.getCatalog());
DataSetScan mainScan=info.getCatalog().getDeclaredDataSetScan().iterator().next();
CatalogBean defaultBean=new CatalogBean(mainScan.getName(),mainScan.getLocation(),true);
toReturn.remove(defaultBean);
toReturn.add(defaultBean);
//*** Cleaning :
// absolute paths to relative paths (from thredds persistence)
// leading/ending '/'
String threddsPersistencePath=info.getLocalBasePath();
for(CatalogBean bean:toReturn) {
String path=bean.getPath();
if(path.startsWith(threddsPersistencePath))
path=path.substring(threddsPersistencePath.length());
if(path.startsWith("/")) path=path.substring(1);
if(path.endsWith("/"))path=path.substring(0, path.length()-1);
bean.setPath(path);
}
return toReturn;
}
private static HashSet<CatalogBean> asCatalogBeanSet(ThreddsCatalog catalog){
HashSet<CatalogBean> toReturn=new HashSet<>();
for(DataSetScan scan:catalog.getDeclaredDataSetScan())
toReturn.add(new CatalogBean(scan.getName(),
scan.getLocation(),false));
if(catalog.getSubCatalogs()!=null&&catalog.getSubCatalogs().getLinkedCatalogs()!=null)
for(ThreddsCatalog sub:catalog.getSubCatalogs().getLinkedCatalogs())
toReturn.addAll(asCatalogBeanSet(sub));
return toReturn;
}
}

318
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java
Unescape Escape View File

@ -0,0 +1,318 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.util.Set;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.tika.io.IOUtils;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.library.client.AuthorizationFilter;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.spatial.data.sdi.model.ServiceConstants;
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.TokenSetter;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.faults.UnableToLockException;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ThreddsController {
private String hostname;
private String operatingPath;
private String targetToken;
public ThreddsController(String path,String targetToken) throws InternalException {
operatingPath=path;
this.targetToken=targetToken;
setTargetToken();
hostname=getThreddsHost();
if(hostname==null) throw new InternalException("Unable to find a thredds instance in target scope "+ScopeUtils.getCurrentScope());
resetCallerToken();
}
private static final String truncate(String toTruncate) {
return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"...";
}
private String callerToken=null;
private void setTargetToken() {
if(callerToken==null) {
callerToken=TokenSetter.getCurrentToken();
log.trace("Storing caller token {}. Target Token is {}",truncate(callerToken),truncate(targetToken));
TokenSetter.setToken(targetToken);
}else {
log.trace("Caller token {} already registered. Target Token is {}",truncate(callerToken),truncate(targetToken));
}
}
private void resetCallerToken() {
if(callerToken!=null) {
log.trace("Resetting caller token {}. Target Token is {}, current is {} ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken()));
TokenSetter.setToken(callerToken);
callerToken=null;
}else log.trace("Caller token {} already reset [current token {}]. Target Token is {}",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken));
}
public final ThreddsInfo getThreddsInfo() {
setTargetToken();
try{
String infoPath="https://"+hostname+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";;
log.info("Loading thredds info from {} ",infoPath);
WebTarget target=getWebClient().target(infoPath);
return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class);
}finally {
resetCallerToken();
}
}
public void lockFolder(String processId) throws UnableToLockException {
setTargetToken();
PrintWriter writer=null;
File temp=null;
try{
log.info("Locking remote path {} to processId {} ",operatingPath,processId);
DataTransferClient cl=getDTClient(hostname);
Destination dest=new Destination();
dest.setCreateSubfolders(true);
dest.setOnExistingFileName(DestinationClashPolicy.FAIL);
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
dest.setPersistenceId(Constants.THREDDS_PERSISTENCE);
dest.setSubFolder(operatingPath);
dest.setDestinationFileName(Constants.LOCK_FILE);
temp=File.createTempFile("tmp_lock", ".tmp");
writer=new PrintWriter(temp);
writer.write(processId);
writer.flush();
writer.close();
cl.localFile(temp, dest);
}catch(Throwable t) {
throw new UnableToLockException("Unable to lock "+operatingPath,t);
}finally {
if(writer!=null) IOUtils.closeQuietly(writer);
if(temp!=null)try {
Files.deleteIfExists(temp.toPath());
}catch(IOException e) {
log.warn("Unable to delete temp file {} ",temp.getAbsolutePath(),e);
}
resetCallerToken();
}
}
public void deleteThreddsFile(String location) throws RemoteFileNotFoundException {
setTargetToken();
String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
log.info("Deleting file at {} ",urlString);
try{
getWebClient().target(urlString).request().delete();
}catch(Throwable t) {
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
}finally{
resetCallerToken();
}
}
public boolean existsThreddsFile(String location) {
try{
getFileDescriptor(location);
return true;
}catch(RemoteFileNotFoundException e) {
return false;
}
}
public String readThreddsFile(String location) throws RemoteFileNotFoundException {
setTargetToken();
String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
log.info("Reading file at {} ",urlString);
try{
return getWebClient().target(urlString).request().get().readEntity(String.class);
}catch(Throwable t) {
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
}finally {
resetCallerToken();
}
}
public void createEmptyFolder(String targetPath) throws InternalException {
setTargetToken();
String toCleanPath=getPathFromStartingLocation(targetPath);
try{
log.info("Cleaning up {} on {} ",toCleanPath,hostname);
DataTransferClient client=getDTClient(hostname);
File toTransfer=File.createTempFile("clean", ".dt_temp");
toTransfer.createNewFile();
Destination dest=new Destination();
dest.setCreateSubfolders(true);
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
dest.setPersistenceId(Constants.THREDDS_PERSISTENCE);
dest.setSubFolder(toCleanPath);
dest.setDestinationFileName(toTransfer.getName());
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
client.localFile(toTransfer, dest);
this.deleteThreddsFile(targetPath+"/"+toTransfer.getName());
log.info("Done");
log.debug("Resulting folder descriptor : {} ",getFileDescriptor(targetPath));
}catch(Exception e) {
log.error("Unable to delete remote folder "+toCleanPath,e);
throw new InternalException("Unable to cleanup remote folder.");
}finally {
resetCallerToken();
}
}
public ThreddsCatalog createCatalog(String name) throws InternalException {
setTargetToken();
try{
log.info("Creating catalog with name {} for path {} ",name,operatingPath);
String sdiUrl="http://"+getSDIServiceHost()+"/"+Constants.SDI_THREDDS_BASE_URL;
Response resp=getWebClient().target(sdiUrl).
queryParam("name", name).
queryParam("path", operatingPath).
queryParam("folder", operatingPath).request().put(null);
if(!(resp.getStatus()>=200&&resp.getStatus()<300))
throw new InternalException("Failed catalog registration on SDI Service. Message "+resp.readEntity(String.class));
return getCatalog();
}catch(Throwable t) {
log.error("Unable to create catalog",t);
throw new InternalException("Unable to create catalog",t);
}finally {
resetCallerToken();
}
}
public ThreddsCatalog getCatalog() {
setTargetToken();
try{ThreddsInfo info=getThreddsInfo();
String instanceBasePath=info.getLocalBasePath();
return info.getCatalogByFittingLocation(instanceBasePath+"/"+operatingPath);
}finally {
resetCallerToken();
}
}
public RemoteFileDescriptor getFileDescriptor() throws RemoteFileNotFoundException {
return getFileDescriptor(null);
}
public RemoteFileDescriptor getFileDescriptor(String path) throws RemoteFileNotFoundException {
setTargetToken();
String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
log.info("Reading file at {} ",urlString);
try{
return getWebClient().target(urlString).queryParam("descriptor", true).request().get().readEntity(RemoteFileDescriptor.class);
}catch(Throwable t) {
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
}finally {
resetCallerToken();
}
}
public InputStream getInputStream(String path) throws RemoteFileNotFoundException {
setTargetToken();
String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
log.info("Reading file at {} ",urlString);
try{
return getWebClient().target(urlString).request().get().readEntity(InputStream.class);
}catch(Throwable t) {
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
}finally {
resetCallerToken();
}
}
public TransferResult transferFile(Destination dest,String url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
setTargetToken();
try{DataTransferClient client=getDTClient(hostname);
if(invocations!=null&&!invocations.isEmpty())
return client.httpSource(url, dest,invocations);
else return client.httpSource(url, dest);
}finally {
resetCallerToken();
}
}
private String getPathFromStartingLocation(String location) {
if(location!=null&&location.length()>0)return operatingPath+"/"+location;
else return operatingPath;
}
private static String getSDIServiceHost(){
return getGCoreEndpointHostname(ServiceConstants.SERVICE_CLASS, ServiceConstants.SERVICE_NAME);
}
private static String getThreddsHost(){
return getGCoreEndpointHostname(ServiceConstants.SERVICE_CLASS, "Thredds");
}
private static String getGCoreEndpointHostname(String serviceClass,String serviceName) {
SimpleQuery query =queryFor(GCoreEndpoint.class);
query.addCondition("$resource/Profile/ServiceClass/text() eq '"+serviceClass+"'")
.addCondition("$resource/Profile/ServiceName/text() eq '"+serviceName+"'");
// .setResult("$resource/Profile/AccessPoint");
DiscoveryClient<GCoreEndpoint> client = clientFor(GCoreEndpoint.class);
GCoreEndpoint endpoint= client.submit(query).get(0);
return endpoint.profile().endpoints().iterator().next().uri().getHost();
}
private static Client getWebClient() {
return ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class))
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
}
private static DataTransferClient getDTClient(String threddsHostName) throws UnreachableNodeException, ServiceNotFoundException {
log.debug("Getting DT Client for {} ",threddsHostName);
return DataTransferClient.getInstanceByEndpoint("http://"+threddsHostName+":80");
}
}

351
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java
Unescape Escape View File

@ -0,0 +1,351 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.io.File;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.threads.ProcessIdProvider;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.LockNotOwnedException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo;
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class WorkspaceFolderManager {
public static SynchronizedElementInfo getInfo(String elementId) {
// TODO FILL THIS
return null;
}
private WorkspaceFolder theFolder;
private String folderId;
// Cahced objects
private SynchFolderConfiguration config=null;
private ThreddsController threddsController=null;
private Workspace ws;
public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException {
try{
ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
theFolder=(WorkspaceFolder) ws.getItem(folderId);
this.folderId=folderId;
}catch(WorkspaceException | InternalErrorException | HomeNotFoundException | UserNotFoundException e) {
throw new WorkspaceInteractionException("Unable to access folder id "+folderId,e);
}
}
public WorkspaceFolder getTheFolder() {
return theFolder;
}
public ThreddsController getThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
if(threddsController==null) {
SynchFolderConfiguration config=getSynchConfiguration();
threddsController=new ThreddsController(config.getRemotePath(),config.getTargetToken());
}
return threddsController;
}
private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
try {
WorkspaceFolder root=(WorkspaceFolder) ws.getItem(getSynchConfiguration().getRootFolderId());
SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root);
return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken());
}catch(WorkspaceException | InternalErrorException e) {
throw new WorkspaceInteractionException(e);
}
}
public boolean isRoot() throws WorkspaceNotSynchedException, WorkspaceInteractionException{
try{
return getSynchConfiguration().getRootFolderId().equals(theFolder.getId());
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException(e);
}
}
public SynchFolderConfiguration getSynchConfiguration() throws WorkspaceInteractionException, WorkspaceNotSynchedException {
if(config==null) {
try {
if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched.");
log.debug("Loading properties for ");
config=WorkspaceUtils.loadConfiguration(theFolder);
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e);
}
}
return config;
}
public boolean isSynched() throws WorkspaceInteractionException {
try{
Map<String,String> props=theFolder.getProperties().getProperties();
return props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null);
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException("Unable to check Synch flag on "+folderId,e);
}
}
public SyncFolderDescriptor check(boolean recursively) throws WorkspaceInteractionException, InternalException {
if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched.");
if(isLocked()&&!isLockOwned())throw new WorkspaceLockedException("Workspace "+folderId+" is locked.");
SynchFolderConfiguration config=getSynchConfiguration();
try{
checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder));
return new SyncFolderDescriptor(this.folderId,this.theFolder.getPath(),config);
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException(e);
}
}
private boolean isLockOwned() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
String currentProcessId=ProcessIdProvider.instance.get();
if(currentProcessId==null) return false;
return currentProcessId.equals(getLockId());
}
public String getLockId() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
return getRootThreddsController().readThreddsFile(Constants.LOCK_FILE);
}
public boolean isLocked() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
return getRootThreddsController().existsThreddsFile(Constants.LOCK_FILE);
}
public void configure(SynchFolderConfiguration toSet) throws WorkspaceInteractionException, InternalException {
if(isSynched()) throw new WorkspaceInteractionException("Folder "+folderId+" is already configured for synchronization.");
log.info("Configuring folder {} as {} ",folderId,toSet);
// Checking AND initializing remote folder
log.debug("Checking remote folder existence .. ");
boolean createCatalog=false;
try {
String catalogName=toSet.getToCreateCatalogName();
ThreddsController controller= new ThreddsController(toSet.getRemotePath(),toSet.getTargetToken());
if(!controller.existsThreddsFile(null)) {
log.info("Folder not found, creating it..");
controller.createEmptyFolder(null);
createCatalog=true;
}else {
ThreddsCatalog catalog=controller.getCatalog();
if (catalog==null) {
createCatalog=true;
}else {
log.info("Found matching catalog {} ",catalog);
catalogName=catalog.getTitle();
if(catalogName==null) catalogName=catalog.getDeclaredDataSetScan().iterator().next().getName();
toSet.setToCreateCatalogName(catalogName);
}
}
if(createCatalog) {
log.info("Creating catalog {} ",catalogName);
log.debug("Created catalog {}", controller.createCatalog(catalogName));
}
WorkspaceUtils.initProperties(theFolder, toSet.getRemotePath(), toSet.getFilter(),
toSet.getTargetToken(),toSet.getToCreateCatalogName(),toSet.getValidateMetadata(),theFolder.getId());
}catch(InternalException e) {
throw new InternalException ("Unable to check/initialize remote folder",e);
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException("Unable to set Properties to "+folderId,e);
}
}
public void dismiss(boolean deleteRemote) throws WorkspaceInteractionException, InternalException {
if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched.");
if(isLocked()&&!isLockOwned())throw new WorkspaceLockedException("Workspace "+folderId+" is locked.");
try {
cleanCache();
WorkspaceUtils.cleanItem(theFolder);
if(deleteRemote)
getThreddsController().createEmptyFolder(null);
}catch(InternalErrorException e) {
throw new WorkspaceInteractionException("Unable to cleanup "+folderId,e);
}
}
public void setLastUpdateTime() throws InternalErrorException {
WorkspaceUtils.setLastUpdateTime(theFolder, System.currentTimeMillis());
}
public void forceUnlock() throws InternalException,WorkspaceInteractionException {
try {
getRootThreddsController().deleteThreddsFile(Constants.LOCK_FILE);
} catch (RemoteFileNotFoundException e) {
log.debug("Forced unlock but no file found.",e);
} catch (WorkspaceNotSynchedException e) {
log.warn("Invoked force lock on not synched folder.",e);
} catch (InternalException | WorkspaceInteractionException e) {
throw e ;
}
}
public void lock(String processId) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
getRootThreddsController().lockFolder(processId);
}
public void unlock() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
unlock(ProcessIdProvider.instance.get());
}
public void unlock(String processId) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
String currentLock=getLockId();
if(processId.equals(currentLock)) getRootThreddsController().deleteThreddsFile(Constants.LOCK_FILE);
else throw new LockNotOwnedException("Process "+processId+" can't remove lock owned by "+currentLock);
}
//*************************** PRIVATE
private void cleanCache() {
this.config=null;
this.threddsController=null;
}
private static void checkFolder(WorkspaceFolder folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws InternalErrorException, InternalException {
// Check folder configuration
log.trace("Checking folder {} ",folder.getPath());
log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder);
String folderName=folder.getName();
String currentRemotePath=rootConfig.getRemotePath()+((relativePathFromRootFolder==null)?"":"/"+relativePathFromRootFolder);
ThreddsController controller=new ThreddsController(currentRemotePath, rootConfig.getTargetToken());
HashSet<String> currentFolderExistingItem=new HashSet<String>();
log.debug("Initializing properties for {} ",folderName);
//INIT PROPERTIES IF NOT PRESENT
if(!WorkspaceUtils.isConfigured(folder))
WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
for(WorkspaceItem item:folder.getChildren()) {
String itemName=item.getName();
String itemRelativePath=(relativePathFromRootFolder==null)?itemName:relativePathFromRootFolder+"/"+itemName;
String itemRemotePath=currentRemotePath+"/"+itemName;
if(item.isFolder()) {
if(recursive)
checkFolder((WorkspaceFolder) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine);
else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
}else if(rootConfig.matchesFilter(itemName)) {
if(!WorkspaceUtils.isConfigured(item))
WorkspaceUtils.initProperties(item, null, null, null,null,null,null);
}
currentFolderExistingItem.add(itemName);
}
// ACTUALLY CHECK STATUS
if(controller.existsThreddsFile(null)) {
SynchronizationStatus folderStatus=SynchronizationStatus.OUTDATED_REMOTE;
log.debug("Remote Folder {} exists. Checking status..",currentRemotePath);
RemoteFileDescriptor folderDesc=controller.getFileDescriptor();
HashSet<String> remoteFolderItems=new HashSet<>(folderDesc.getChildren());
// CHECK HISTORY
Set<String> accountingEntries=WorkspaceUtils.scanAccountingForStatus(folder, rootConfig,
currentFolderExistingItem, remoteFolderItems,
controller, null, null);
if(accountingEntries.isEmpty()) {
log.debug("No accounting entries found");
folderStatus=SynchronizationStatus.UP_TO_DATE;
}
// CHECK WS ITEMS
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder()||rootConfig.matchesFilter(item.getName())) {
SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item, remoteFolderItems, controller,lastUpdatedRoutine);
item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+""));
folderStatus=folderStatus.equals(SynchronizationStatus.UP_TO_DATE)?itemStatus:folderStatus;
}
// CHECK REMOTE FOLDER
if(folderStatus.equals(SynchronizationStatus.UP_TO_DATE)) {
Set<String> toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, accountingEntries, currentFolderExistingItem, folder, controller, rootConfig, null, null);
if(!toImportItems.isEmpty()) folderStatus=SynchronizationStatus.OUTDATED_WS;
}
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+""));
}else {
// Remote Folder not existing, set everything to OUTDATED_REMOTE
for(WorkspaceItem item:folder.getChildren())
item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""));
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+""));
}
}
public File loadCatalogFile() {
// TODO Auto-generated method stub
return null;
}
public void updateCatalogFile(File toUpload) {
// TODO Auto-generated method stub
}
}

356
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java
Unescape Escape View File

@ -0,0 +1,356 @@
package org.gcube.usecases.ws.thredds.engine.impl;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibary.model.items.type.WorkspaceItemType;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.Properties;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRemoval;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRenaming;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest;
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class WorkspaceUtils {
/**
* Checks current accounting info in order to infere synchronization status.
* OwnerProcess and service parameters can be null for check purposes.
*
* @param folderPath
* @param toScanFolder
* @param config
* @param localChildrenNames
* @param remoteChildrenNames
* @param folderController
*
* @param ownerProcess
* @param service
* @return set of Item names that have been found to be synchronized
* @throws InternalErrorException
*/
static Set<String> scanAccountingForStatus(
WorkspaceFolder toScanFolder,
SynchFolderConfiguration config,
Set<String> localChildrenNames,
Set<String> remoteChildrenNames,
ThreddsController folderController,
Process ownerProcess,
ExecutorService service) throws InternalErrorException{
Set<String> handledAccountingEntries=new HashSet<>();
log.debug("Checking history of {} ",toScanFolder.getPath());
String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
Date folderLastUpdateTime=null;
try{
folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolder);
}catch(Throwable t) {
log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolder.getName(),t);
folderLastUpdateTime=new Date(0l);
}
// scanning for deletions
log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolder.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime));
for(AccountingEntry entry:toScanFolder.getAccounting()) {
try {
Date eventTime=entry.getDate().getTime();
if(folderLastUpdateTime==null|| eventTime.after(folderLastUpdateTime)) { // SKIP IF ENTRY OLDER THAN LAST UPDATE TIME
String toDeleteRemote=null;
switch(entry.getEntryType()) {
case CUT:
case REMOVAL:{
AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry;
if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER)||
config.matchesFilter(removalEntry.getItemName()))
toDeleteRemote=removalEntry.getItemName();
break;
}
case RENAMING:{
AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry;
WorkspaceItem newItem=toScanFolder.find(renamingEntry.getNewItemName());
if(newItem.isFolder()||config.matchesFilter(renamingEntry.getOldItemName()))
toDeleteRemote=renamingEntry.getOldItemName();
break;
}
}
if(toDeleteRemote!=null){
// SKIP IF LOCAL EXISTS
if(localChildrenNames.contains(toDeleteRemote))
log.debug("Skipping accounting entry for existing local item {} ",toDeleteRemote);
else if(remoteChildrenNames.contains(toDeleteRemote)) {
log.debug("Checking age of remote {} ",toDeleteRemote);
// IF REMOTE OLDER THAN ENTRY -> DELETE REQUEST
// IF REMOTE NEWER -> IMPORT REQUEST
RemoteFileDescriptor remote=folderController.getFileDescriptor(relativePath+"/"+toDeleteRemote);
Date remoteDate=new Date(remote.getLastUpdate());
log.debug("Last remote update : {} . Event date {} ",Constants.DATE_FORMAT.format(remoteDate),Constants.DATE_FORMAT.format(eventTime));
if(service!=null) {
log.debug("Service is not null. Submitting request ... ");
if(eventTime.after(remoteDate)) {
service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolder,toDeleteRemote)));
handledAccountingEntries.add(toDeleteRemote);
log.debug("Submitted DELETION request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
}
// }else {
// service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, toDeleteRemote)));
// log.debug("Submitted UPDATE-LOCAL request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
// }
}
}else log.debug("To delete remote {} not found. skipping it.. ",toDeleteRemote);
// SKIP IF REMOTE NOT FOUND
}
}
}catch(Throwable t) {
log.error("Unable to submit deletion request for {} ",entry,t);
}
}
return handledAccountingEntries;
}
/**
* Scans remote Folder in order to gather elements to be synchronized.
* OwnerProcess and Service can be null for check purposes.
*
*
* @param folderPath
* @param folderDesc
* @param handledAccountingEntries
* @param handledWorkspaceItemEntries
* @param toScanFolder
* @param folderController
* @param config
* @param ownerProcess
* @param service
* @return
* @throws InternalException
* @throws InternalErrorException
*/
static Set<String> scanRemoteFolder(
RemoteFileDescriptor folderDesc,
Set<String> handledAccountingEntries,
Set<String> handledWorkspaceItemEntries,
WorkspaceFolder toScanFolder,
ThreddsController folderController,
SynchFolderConfiguration config,
Process ownerProcess,
ExecutorService service) throws InternalException, InternalErrorException{
log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanFolder.getPath(),folderDesc.getAbsolutePath());
Set<String> handledRemoteElements=new HashSet<String>();
// String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
if(!folderDesc.isDirectory()) throw new InternalException("Remote Descriptor "+folderDesc.getAbsolutePath()+" Is not a directory. ");
for(String child:folderDesc.getChildren()) {
// skip if already handled with accounting
if(handledAccountingEntries.contains(child))
log.debug("Skipping remote child {} because already handled with accouting", child);
// skip if already handled with local items
else if(handledWorkspaceItemEntries.contains(child))
log.debug("Skipping remote child {} because already handled with respective item",child);
else {
RemoteFileDescriptor childDesc=folderController.getFileDescriptor(child);
if(childDesc.isDirectory()) {
handledRemoteElements.add(child);
}else if (config.matchesFilter(child)){
log.debug("Child {} matches filter...");
handledRemoteElements.add(child);
if(service!=null) {
service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, child)));
log.debug("Submitted IMPORT request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet());
}
// import if matching
}else log.debug("Skipping not matching remote {} ",child);
// skip if doesn't match filter or isn't folder
}
}
return handledRemoteElements;
}
static void initProperties(WorkspaceItem toInit, String remotePath, String filter, String targetToken,
String catalogName,Boolean validateMeta, String rootFolderId) throws InternalErrorException {
Map<String,String> toSetProperties=toInit.getProperties().getProperties();
initIfMissing(toSetProperties,Constants.WorkspaceProperties.TBS,"true");
initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_TIME,0l+"");
initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_STATUS,StepReport.Status.OK+"");
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.UP_TO_DATE+"");
if(toInit.isFolder()) {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.TARGET_TOKEN,targetToken);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.RELATED_CATALOG,catalogName);
initIfMissing(toSetProperties,Constants.WorkspaceProperties.VALIDATE_METADATA,validateMeta+"");
initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId);
}else {
initIfMissing(toSetProperties,Constants.WorkspaceProperties.METADATA_UUID,null);
}
toInit.getProperties().addProperties(toSetProperties);
}
private static void initIfMissing(Map<String,String> current,String key,String defaultValue) {
if(!current.containsKey(key)||
current.get(key)==null||
current.get(key).equals("null")) current.put(key, defaultValue);
}
static boolean isConfigured(WorkspaceItem toCheck) throws InternalErrorException {
return isConfigured(toCheck.getProperties().getProperties());
}
static boolean isConfigured(Map<String,String> toCheckProperties) {
return (toCheckProperties.containsKey(Constants.WorkspaceProperties.TBS)&&toCheckProperties.get(Constants.WorkspaceProperties.TBS)!=null);
}
static SynchronizationStatus getStatusAgainstRemote(WorkspaceItem item, Set<String> existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException, InternalErrorException, RemoteFileNotFoundException {
String itemName=item.getName();
SynchronizationStatus status=SynchronizationStatus.OUTDATED_REMOTE;
if(existingRemote.contains(itemName)) {
RemoteFileDescriptor desc=remoteFolderController.getFileDescriptor(itemName);
Date remoteDate=new Date(desc.getLastUpdate());
Date localDate=item.getLastModificationTime().getTime();
Date lastUpdate=safelyGetLastUpdate(item);
if(localDate.equals(lastUpdate)) {
//LAST MODIFCATION WAS FROM SYNCHRONIZATION
if(remoteDate.after(lastUpdate)) status=SynchronizationStatus.OUTDATED_WS;
else status=SynchronizationStatus.UP_TO_DATE;
}else
if(remoteDate.before(localDate)) { // REMOTE OLDER THAN LOCAL
if(isModifiedAfter(item,lastUpdateRoutine)) status=SynchronizationStatus.OUTDATED_REMOTE; // IT's been locally modified from last routine
else status=SynchronizationStatus.UP_TO_DATE;
}
else if(remoteDate.after(localDate)) { // REMOTE NEWER &..
if (remoteDate.equals(lastUpdate))status =SynchronizationStatus.UP_TO_DATE; // REMOTE DATE == LAST UPDATE ROUTINE -> UP TO DATE
else if (remoteDate.before(lastUpdate))status =SynchronizationStatus.OUTDATED_REMOTE; // REMOTE DATE < LAST UPDATE -> transfer to thredds, last update was faulty
else status=SynchronizationStatus.OUTDATED_WS; // REMOTE DATE != LAST UPDATE -> import from thredds
}
}
return status;
}
//
// /**
// *
// * @return max date between creation time, last modification time && LAST-UPDATE-PROP
// * @throws InternalErrorException
// * @throws NumberFormatException
// */
// static Date getMaxLastUpdate(WorkspaceItem item) throws NumberFormatException, InternalErrorException {
// return new Date(Long.max(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)),item.getLastModificationTime().getTimeInMillis()));
// }
static Date safelyGetLastUpdate(WorkspaceItem item) throws InternalErrorException {
try {
return new Date(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)));
}catch(NumberFormatException e) {
log.debug("Unable to get last update time for {} ",item.getName(),e);
return new Date(0l);
}
}
public static boolean isModifiedAfter(WorkspaceItem item,Date fromDate) throws InternalErrorException {
for(AccountingEntry entry:item.getAccounting()) {
if(entry.getDate().getTime().after(fromDate)) {
switch(entry.getEntryType()) {
case PASTE:
case CREATE:
case RESTORE:
case UPDATE:
case ADD: return true;
}
}
}
return false;
}
static void cleanItem(WorkspaceItem item) throws InternalErrorException {
Properties props=item.getProperties();
if(props.hasProperty(Constants.WorkspaceProperties.TBS)) {
if(item.isFolder()) {
props.addProperties(Constants.cleanedFolderPropertiesMap);
for(WorkspaceItem child : ((WorkspaceFolder)item).getChildren())
cleanItem(child);
}else props.addProperties(Constants.cleanedItemPropertiesMap);
}
}
static void setLastUpdateTime(WorkspaceFolder folder,long toSetTime) throws InternalErrorException {
StepReport.Status currentWSStatus=StepReport.Status.valueOf(folder.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_STATUS));
if(currentWSStatus.equals(StepReport.Status.OK))
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+""));
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder()) setLastUpdateTime((WorkspaceFolder) item, toSetTime);
}
public static SynchFolderConfiguration loadConfiguration(WorkspaceItem item) throws InternalErrorException {
if(item.isFolder()) {
Properties props=item.getProperties();
SynchFolderConfiguration config=new SynchFolderConfiguration();
config.setFilter(props.getPropertyValue(Constants.WorkspaceProperties.SYNCH_FILTER));
config.setRemotePath(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH));
config.setRemotePersistence(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
config.setTargetToken(props.getPropertyValue(Constants.WorkspaceProperties.TARGET_TOKEN));
config.setToCreateCatalogName(props.getPropertyValue(Constants.WorkspaceProperties.RELATED_CATALOG));
config.setValidateMetadata(Boolean.parseBoolean(props.getPropertyValue(Constants.WorkspaceProperties.VALIDATE_METADATA)));
config.setRootFolderId(props.getPropertyValue(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
return config;
}else return loadConfiguration(item.getParent());
}
static void resetStatus(WorkspaceItem item) throws InternalErrorException {
if(item.isFolder()) {
for(WorkspaceItem child: ((WorkspaceFolder)item).getChildren())
resetStatus(child);
}
Map<String,String> props=item.getProperties().getProperties();
if(props.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) {
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+"");
item.getProperties().addProperties(props);
}
}
}

18
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java
Unescape Escape View File

@ -0,0 +1,18 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
@Data
public class DeleteRemoteRequest extends SynchronizationRequest {
private String toRemoveName;
public DeleteRemoteRequest(Process process,WorkspaceFolder location,String name) {
super(process,location);
this.toRemoveName=name;
}
}

30
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessIdProvider.java
Unescape Escape View File

@ -0,0 +1,30 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
public class ProcessIdProvider {
public static ProcessIdProvider instance=new ProcessIdProvider();
private static final InheritableThreadLocal<String> threadProcessId=
new InheritableThreadLocal<String>() {
@Override
protected String initialValue() {
return null;
}
};
private ProcessIdProvider() {
}
public String get() {
return threadProcessId.get();
}
public void set(String processId) {
threadProcessId.set(processId);
}
public void reset() {
threadProcessId.remove();
}
}

59
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java
Unescape Escape View File

@ -0,0 +1,59 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.util.concurrent.ExecutorService;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import lombok.Data;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Data
@RequiredArgsConstructor
@Slf4j
public class ProcessInitializationThread implements Runnable {
@NonNull
private Process theProcess;
@NonNull
private ExecutorService service;
@Override
public void run() {
ProcessDescriptor descriptor=theProcess.getDescriptor();
ProcessIdProvider.instance.set(descriptor.getProcessId());
log.info("Initialization of process {} ",descriptor);
theProcess.getStatus().setCurrentMessage("Gathering synchronization information...");
try {
WorkspaceFolderManager manager=new WorkspaceFolderManager(descriptor.getFolderId());
log.debug("Updateing synchronization status..");
manager.check(true);
log.debug("Launching requests...");
theProcess.launch(service);
}catch(WorkspaceInteractionException e) {
log.error("Unable to proceed..",e);
theProcess.cancel();
} catch (InternalException e) {
log.error("Unable to proceed..",e);
theProcess.cancel();
} catch (InternalErrorException e) {
log.error("Unable to proceed..",e);
theProcess.cancel();
}catch(Throwable t) {
log.error("Unexpected Error : ",t);
theProcess.cancel();
}finally {
ProcessIdProvider.instance.reset();
}
}
}

113
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java
Unescape Escape View File

@ -0,0 +1,113 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.io.BufferedWriter;
import java.io.FileWriter;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.SyncEngine;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class RequestLogger {
private static RequestLogger instance=null;
@Synchronized
public static RequestLogger get() {
if(instance==null) {
instance=new RequestLogger();
}
return instance;
}
BufferedWriter writer = null;
private RequestLogger() {
SyncEngine engine=SyncEngine.get();
if(engine.isRequestLoggerEnabled()) {
String path=engine.getRequestLoggerPath();
log.info("Initializing Request Logger to path {} ",path);
try{
writer = new BufferedWriter(new FileWriter(path, false));
}catch(Throwable t) {
log.warn("Unable to initialize Writer on path {} ",path,t);
}
}
}
public void log(SynchronizationRequest request) {
if(writer!=null) {
String logString="INVALID";
if(request instanceof DeleteRemoteRequest) {
DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) request;
logString=String.format("DELETE %s from %s (WS-Path: %s)", deleteRequest.getToRemoveName(),getRemotePath(deleteRequest.getLocation()),getWSPath(deleteRequest.getLocation()));
}else if(request instanceof TransferToThreddsRequest) {
TransferToThreddsRequest transferRequest=(TransferToThreddsRequest) request;
logString=String.format("EXPORT %s to %s from %s", getName(transferRequest.getToTransfer()),getRemotePath(transferRequest.getToTransfer()),getWSPath(transferRequest.getToTransfer()));
}else if(request instanceof TransferFromThreddsRequest) {
TransferFromThreddsRequest transferRequest=(TransferFromThreddsRequest)request;
if(transferRequest.getTargetItem()!=null)
logString=String.format("UPDATE LOCAL %s in %s from %s ", getName(transferRequest.getTargetItem()),getWSPath(transferRequest.getLocation()),getRemotePath(transferRequest.getLocation()));
else
logString=String.format("IMPORT LOCAL %s in %s from %s ", transferRequest.getRemoteFilename(),getWSPath(transferRequest.getLocation()),getRemotePath(transferRequest.getLocation()));
}
try {
writer.write(logString+"\n");
}catch(Throwable t) {
log.warn("Exception wile trying to write log",t);
}
}
}
public void close() {
if(writer!=null) {
try {
writer.flush();
writer.close();
}catch(Throwable t) {
log.warn("Unable to close writer ",t);
}
}
}
private static final String getRemotePath(WorkspaceItem item) {
try{
if(item.isFolder()) return item.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
else return getRemotePath(item.getParent());
}catch(InternalErrorException e) {
log.warn("Unable to get Remote Path ",e);
return "N/A";
}
}
private static final String getName(WorkspaceItem item) {
try {
return item.getName();
}catch(InternalErrorException e) {
log.warn("Unable to get name ",e);
return "N/A";
}
}
private static final String getWSPath(WorkspaceItem item) {
try{
if(item.isFolder()) return item.getPath();
else return getRemotePath(item.getParent());
}catch(InternalErrorException e) {
log.warn("Unable to get WS Path ",e);
return "N/A";
}
}
}

16
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java
Unescape Escape View File

@ -0,0 +1,16 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
import lombok.NonNull;
@Data
public abstract class SynchronizationRequest {
@NonNull
private Process process;
@NonNull
private WorkspaceFolder location;
}

289
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java
Unescape Escape View File

@ -0,0 +1,289 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import java.io.File;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.ExecutionReport;
import org.gcube.data.transfer.model.ExecutionReport.ExecutionReportFlag;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.data.transfer.model.RemoteFileDescriptor;
import org.gcube.data.transfer.model.plugins.thredds.DataSet;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.usecases.ws.thredds.Constants;
import org.gcube.usecases.ws.thredds.NetUtils;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
import org.gcube.usecases.ws.thredds.faults.CancellationException;
import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
import org.gcube.usecases.ws.thredds.model.StepReport;
import org.gcube.usecases.ws.thredds.model.StepReport.OperationType;
import org.gcube.usecases.ws.thredds.model.StepReport.Status;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SynchronizationThread implements Runnable {
private SynchronizationRequest theRequest;
public SynchronizationThread(SynchronizationRequest theRequest) {
super();
this.theRequest = theRequest;
}
@Override
public void run() {
ProcessIdProvider.instance.set(theRequest.getProcess().getDescriptor().getProcessId());
log.debug("Executing Transfer request {} ",theRequest);
RequestLogger.get().log(theRequest);
String reportMessage="Never started";
String reportItemName="Still Unknown";
Status toSetStatus=Status.ERROR;
WorkspaceFolder parentFolder=theRequest.getLocation();
try {
checkCancelledProcess();
SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(theRequest.getLocation());
ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken());
if(theRequest instanceof TransferToThreddsRequest) {
TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest;
WorkspaceItem item=request.getToTransfer();
//look for metadata in same folder
String itemName=item.getName();
reportItemName=itemName;
String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml";
WorkspaceItem metadataItem=getFileByName(item.getParent(),false,toLookMetadataName);
// if not present, generate with sis/geotk
Destination toSetDestination=new Destination();
toSetDestination.setCreateSubfolders(true);
toSetDestination.setDestinationFileName(itemName);
toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE);
toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
toSetDestination.setPersistenceId(synchConfig.getRemotePersistence());
//NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
String fileLocation=request.getLocation().getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH);
toSetDestination.setSubFolder(fileLocation);
checkCancelledProcess();
Set<PluginInvocation> invocations=null;
if(metadataItem==null) {
log.debug("Metadata not found, asking SIS/GEOTK for generation..");
invocations=Collections.singleton(new PluginInvocation(Constants.SIS_PLUGIN_ID));
}
log.info("Transferring to {} with invocations {} ",toSetDestination,invocations);
ThreddsInfo info=controller.getThreddsInfo();
DataSet dataset=info.getDataSetFromLocation(info.getLocalBasePath()+"/"+fileLocation);
// ThreddsCatalog catalog=controller.getCatalog();
checkCancelledProcess();
TransferResult result=controller.transferFile(toSetDestination, item.getPublicLink(false), invocations);
Map<String,String> toSetProperties=new HashMap<String,String>();
String toSetMetadataUUID=null;
Boolean validateMetadata=synchConfig.getValidateMetadata();
checkCancelledProcess();
if(metadataItem==null) {
ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID);
if(!report.getFlag().equals(ExecutionReportFlag.SUCCESS)) throw new DataTransferPluginError("Unable to Extract Metadata for "+itemName+" Message is "+report.getMessage());
else toSetMetadataUUID=report.getMessage();
}else {
MetadataReport metaReport=publishMetadata(metadataItem,info.getHostname(),itemName,dataset.getPath(),validateMetadata);
toSetMetadataUUID=metaReport.getPublishedUUID();
}
toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, controller.getFileDescriptor(itemName).getLastUpdate()+"");
toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID);
toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+"");
item.getProperties().addProperties(toSetProperties);
reportMessage="Successfully transferred and published.";
// End ws->th
} else if(theRequest instanceof TransferFromThreddsRequest) {
Map<String,String> toSetProperties=new HashMap<String,String>();
TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest;
String toImportName=null;
WorkspaceItem targetItem=null;
if(importRequest.getTargetItem()==null) {
//Target Item will be created
toImportName=importRequest.getRemoteFilename();
reportMessage="Importing remote file..";
}else {
//Target Item already exists
toImportName=importRequest.getTargetItem().getName();
targetItem=importRequest.getTargetItem();
reportMessage="Updating file..";
}
reportItemName=toImportName;
RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName);
toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+"");
InputStream source=null;
try {
source=controller.getInputStream(toImportName);
if(targetItem==null)
targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source);
else
targetItem.updateItem(source);
targetItem.getProperties().addProperties(toSetProperties);
}finally {
if(source!=null)
source.close();
}
reportMessage="File successfully imported";
}else if(theRequest instanceof DeleteRemoteRequest) {
DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest;
reportItemName=deleteRequest.getToRemoveName();
log.debug("Going to delete {} from ",reportItemName,synchConfig.getRemotePath());
RemoteFileDescriptor desc=controller.getFileDescriptor(reportItemName);
if(desc.isDirectory()) {
log.debug("Remote {} is directory.. Cleaning it up, first.",reportItemName);
controller.createEmptyFolder(null);
}
controller.deleteThreddsFile(reportItemName);
reportMessage="Successfully removed";
}
log.info("Synchronization of {} successful.",reportItemName);
toSetStatus=Status.OK;
}catch(CancellationException e) {
log.debug("Process cancelled.. ",e);
reportMessage="CancelledProcess";
toSetStatus=Status.CANCELLED;
}catch(DataTransferPluginError e) {
log.debug("Unable to extract metadata ",e);
reportMessage="Unable to extract metadata : "+e.getMessage();
toSetStatus=Status.ERROR;
}catch(RemoteFileNotFoundException e) {
log.debug("Remote File not found ",e);
reportMessage="Remote File not found : "+e.getMessage();
toSetStatus=Status.ERROR;
}catch(InternalErrorException e) {
log.debug("Internal generic exception ",e);
reportMessage="Internal error : "+e.getMessage();
toSetStatus=Status.ERROR;
}catch(Throwable t) {
log.debug("Internal generic exception ",t);
reportMessage="Unexpected exception : "+t.getMessage();
toSetStatus=Status.ERROR;
}finally {
updateParentProperty(parentFolder, toSetStatus);
submitReport(reportItemName,reportMessage,toSetStatus);
ProcessIdProvider.instance.reset();
}
}
@Synchronized
private static void updateParentProperty(WorkspaceFolder folder,StepReport.Status toSetStatus) {
try {
String currentValue=folder.getProperties().getProperties().get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS);
if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null"))
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""));
else {
StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue);
if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus))
folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+""));
}
}catch(Throwable t) {
log.warn("Unable to update folder status ",t);
}
}
private void checkCancelledProcess() throws CancellationException{
if(theRequest.getProcess().getStatus().
getStatus().equals(ProcessStatus.Status.STOPPED))
throw new CancellationException("Process "+theRequest.getProcess().getDescriptor().getProcessId()+" has been cancelled");
}
private void submitReport(String elementName,String message,StepReport.Status status) {
StepReport report=new StepReport(elementName,message,status,OperationType.WS_TO_TH,System.currentTimeMillis());
if(theRequest instanceof TransferToThreddsRequest) report.setOperationType(OperationType.WS_TO_TH);
else if(theRequest instanceof TransferFromThreddsRequest) report.setOperationType(OperationType.TH_TO_WS);
else if(theRequest instanceof DeleteRemoteRequest) report.setOperationType(OperationType.DELETE_REMOTE);
else throw new RuntimeException("Unknown operation request "+theRequest);
theRequest.getProcess().onStep(report);
}
private static MetadataReport publishMetadata(WorkspaceItem toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{
File tempMetaFile=null;
try {
Metadata meta=SDIAbstractPlugin.metadata().build();
tempMetaFile=NetUtils.download(toPublish.getPublicLink(false));
log.debug("Publishing metadata {} ",filename);
MetadataPublishOptions opts=new MetadataPublishOptions(
new TemplateInvocationBuilder().threddsOnlineResources(threddsHostname, filename, publicPath).get());
opts.setGeonetworkCategory("Datasets");
opts.setValidate(validate);
return meta.pushMetadata(tempMetaFile, opts);
}catch(Throwable t) {
if(tempMetaFile!=null) Files.deleteIfExists(tempMetaFile.toPath());
throw new Exception("Something went wrong while publishing metadata for "+filename+". Cause : "+t.getMessage(),t);
}
}
private static final WorkspaceItem getFileByName(WorkspaceFolder toLookIntoFolder,boolean recursive,String toLookForName) throws InternalErrorException {
log.debug("Looking for {} into {} [recursive {} ]",toLookForName,toLookIntoFolder.getPath()+" ID "+toLookIntoFolder.getId(),recursive);
for(WorkspaceItem item : toLookIntoFolder.getChildren())
if(!item.isFolder()&&item.getName().equals(toLookForName)) return item;
if(recursive) {
for(WorkspaceItem item : toLookIntoFolder.getChildren())
if(item.isFolder()) return getFileByName((WorkspaceFolder) item, recursive, toLookForName);
}
return null;
}
}

26
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java
Unescape Escape View File

@ -0,0 +1,26 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Data;
@Data
public class TransferFromThreddsRequest extends SynchronizationRequest {
private WorkspaceItem targetItem; // can be null
private String remoteFilename;
public TransferFromThreddsRequest(Process process, WorkspaceItem targetItem, WorkspaceFolder containingFolder,
String remoteFilename) {
super(process,containingFolder);
this.targetItem = targetItem;
this.remoteFilename = remoteFilename;
}
}

23
src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java
Unescape Escape View File

@ -0,0 +1,23 @@
package org.gcube.usecases.ws.thredds.engine.impl.threads;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class TransferToThreddsRequest extends SynchronizationRequest {
private WorkspaceItem toTransfer;
public TransferToThreddsRequest(Process process,WorkspaceFolder location, WorkspaceItem toTransfer) {
super(process,location);
this.toTransfer = toTransfer;
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/CancellationException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class CancellationException extends Exception {
/**
*
*/
private static final long serialVersionUID = 736538914489728352L;
public CancellationException() {
// TODO Auto-generated constructor stub
}
public CancellationException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public CancellationException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public CancellationException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public CancellationException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/DataTransferPluginError.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class DataTransferPluginError extends Exception {
/**
*
*/
private static final long serialVersionUID = 5928532367397696543L;
public DataTransferPluginError() {
// TODO Auto-generated constructor stub
}
public DataTransferPluginError(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public DataTransferPluginError(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public DataTransferPluginError(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public DataTransferPluginError(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

37
src/main/java/org/gcube/usecases/ws/thredds/faults/InternalException.java
Unescape Escape View File

@ -0,0 +1,37 @@
package org.gcube.usecases.ws.thredds.faults;
public class InternalException extends Exception {
/**
*
*/
private static final long serialVersionUID = 3088246150497872471L;
public InternalException() {
super();
// TODO Auto-generated constructor stub
}
public InternalException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public InternalException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public InternalException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public InternalException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/LockNotOwnedException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class LockNotOwnedException extends InternalException {
/**
*
*/
private static final long serialVersionUID = 96212116766290884L;
public LockNotOwnedException() {
// TODO Auto-generated constructor stub
}
public LockNotOwnedException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public LockNotOwnedException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public LockNotOwnedException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public LockNotOwnedException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/ProcessNotFoundException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class ProcessNotFoundException extends Exception {
/**
*
*/
private static final long serialVersionUID = 1L;
public ProcessNotFoundException() {
// TODO Auto-generated constructor stub
}
public ProcessNotFoundException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public ProcessNotFoundException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public ProcessNotFoundException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public ProcessNotFoundException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/RemoteFileNotFoundException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class RemoteFileNotFoundException extends InternalException {
/**
*
*/
private static final long serialVersionUID = -4333525223191661141L;
public RemoteFileNotFoundException() {
// TODO Auto-generated constructor stub
}
public RemoteFileNotFoundException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public RemoteFileNotFoundException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public RemoteFileNotFoundException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public RemoteFileNotFoundException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/UnableToLockException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class UnableToLockException extends InternalException {
/**
*
*/
private static final long serialVersionUID = -6926321942546668032L;
public UnableToLockException() {
// TODO Auto-generated constructor stub
}
public UnableToLockException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public UnableToLockException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public UnableToLockException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public UnableToLockException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceFolderNotRootException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class WorkspaceFolderNotRootException extends WorkspaceInteractionException {
/**
*
*/
private static final long serialVersionUID = 5025376971995804122L;
public WorkspaceFolderNotRootException() {
// TODO Auto-generated constructor stub
}
public WorkspaceFolderNotRootException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public WorkspaceFolderNotRootException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public WorkspaceFolderNotRootException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public WorkspaceFolderNotRootException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

37
src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceInteractionException.java
Unescape Escape View File

@ -0,0 +1,37 @@
package org.gcube.usecases.ws.thredds.faults;
public class WorkspaceInteractionException extends Exception {
/**
*
*/
private static final long serialVersionUID = -1121867242282602650L;
public WorkspaceInteractionException() {
super();
// TODO Auto-generated constructor stub
}
public WorkspaceInteractionException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public WorkspaceInteractionException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public WorkspaceInteractionException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public WorkspaceInteractionException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceLockedException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class WorkspaceLockedException extends WorkspaceInteractionException {
/**
*
*/
private static final long serialVersionUID = 4220255895750039163L;
public WorkspaceLockedException() {
// TODO Auto-generated constructor stub
}
public WorkspaceLockedException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public WorkspaceLockedException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public WorkspaceLockedException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public WorkspaceLockedException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

35
src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceNotSynchedException.java
Unescape Escape View File

@ -0,0 +1,35 @@
package org.gcube.usecases.ws.thredds.faults;
public class WorkspaceNotSynchedException extends WorkspaceInteractionException {
/**
*
*/
private static final long serialVersionUID = -321145123900245553L;
public WorkspaceNotSynchedException() {
// TODO Auto-generated constructor stub
}
public WorkspaceNotSynchedException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public WorkspaceNotSynchedException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public WorkspaceNotSynchedException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public WorkspaceNotSynchedException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

9
src/main/java/org/gcube/usecases/ws/thredds/model/CompletionCallback.java
Unescape Escape View File

@ -0,0 +1,9 @@
package org.gcube.usecases.ws.thredds.model;
import org.gcube.usecases.ws.thredds.engine.impl.Process;
public interface CompletionCallback {
public void onProcessCompleted(Process completedProcess);
}

25
src/main/java/org/gcube/usecases/ws/thredds/model/StepReport.java
Unescape Escape View File

@ -0,0 +1,25 @@
package org.gcube.usecases.ws.thredds.model;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class StepReport {
public static enum Status{
ERROR,OK,CANCELLED
}
public static enum OperationType{
WS_TO_TH,TH_TO_WS,DELETE_REMOTE
}
private String elementName;
private String message;
private Status status;
private OperationType operationType;
private long completionTime;
}

26
src/main/java/org/gcube/usecases/ws/thredds/model/SyncFolderDescriptor.java
Unescape Escape View File

@ -0,0 +1,26 @@
package org.gcube.usecases.ws.thredds.model;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import lombok.Data;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@RequiredArgsConstructor
public class SyncFolderDescriptor {
@NonNull
private String folderId;
@NonNull
private String folderPath;
@NonNull
private SynchFolderConfiguration configuration;
@NonNull
private boolean isLocked=false;
private ProcessDescriptor localProcessDescriptor=null;
}

10
src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationCallBack.java
Unescape Escape View File

@ -0,0 +1,10 @@
package org.gcube.usecases.ws.thredds.model;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
public interface SyncOperationCallBack {
public void onStep(ProcessStatus status, ProcessDescriptor descriptor);
}

5
src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationTicket.java
Unescape Escape View File

@ -0,0 +1,5 @@
package org.gcube.usecases.ws.thredds.model;
public class SyncOperationTicket {
}

37
src/main/java/org/gcube/usecases/ws/thredds/model/SynchFolderConfiguration.java
Unescape Escape View File

@ -0,0 +1,37 @@
package org.gcube.usecases.ws.thredds.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@RequiredArgsConstructor
public class SynchFolderConfiguration {
@NonNull
private String remotePath;
@NonNull
private String filter;
@NonNull
private String targetToken;
@NonNull
private String toCreateCatalogName;
private String remotePersistence="thredds";
@NonNull
private Boolean validateMetadata=true;
@NonNull
private String rootFolderId;
public boolean matchesFilter(String name) {
return name.endsWith(".nc")||name.endsWith(".ncml")||name.endsWith(".asc");
}
}

10
src/main/java/org/gcube/usecases/ws/thredds/model/SynchronizedElementInfo.java
Unescape Escape View File

@ -0,0 +1,10 @@
package org.gcube.usecases.ws.thredds.model;
public class SynchronizedElementInfo {
public static enum SynchronizationStatus{
UP_TO_DATE,OUTDATED_WS,OUTDATED_REMOTE
}
}

43
src/main/java/org/gcube/usecases/ws/thredds/model/gui/CatalogBean.java
Unescape Escape View File

@ -0,0 +1,43 @@
package org.gcube.usecases.ws.thredds.model.gui;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@AllArgsConstructor
@ToString
public class CatalogBean {
private String name;
private String path;
private Boolean isDefault;
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((path == null) ? 0 : path.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
CatalogBean other = (CatalogBean) obj;
if (path == null) {
if (other.path != null)
return false;
} else if (!path.equals(other.path))
return false;
return true;
}
}

11
src/main/resources/log4j.properties
Unescape Escape View File

@ -0,0 +1,11 @@
# Root logger option
#log4j.rootLogger=INFO, SM
log4j.logger.org.gcube.usecases.ws=DEBUG,SM
#SM-specific logger with dedicated appender
log4j.appender.SM=org.apache.log4j.ConsoleAppender
log4j.appender.SM.Target=System.out
log4j.appender.SM.layout=org.apache.log4j.PatternLayout
log4j.appender.SM.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

0
src/main/resources/logback.xml
Unescape Escape View File

6
src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties
Unescape Escape View File

@ -0,0 +1,6 @@
scanner.pool.maxSize=10
scanner.pool.coreSize=1
scanner.pool.idle.ms=30000
transfers.pool.maxSize=40
transfers.pool.coreSize=1
transfers.pool.idle.ms=30000

78
src/test/java/org/gcube/usecases/ws/thredds/DTSynchUseCase.java
Unescape Escape View File

@ -0,0 +1,78 @@
package org.gcube.usecases.ws.thredds;
import java.util.concurrent.Semaphore;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class DTSynchUseCase {
public static void main(String[] args) throws WorkspaceInteractionException, InternalException, ProcessNotFoundException {
TokenSetter.set("/d4science.research-infrastructures.eu");
SyncEngine engine=SyncEngine.get();
String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d";
// String folderId="8a6f9749-68d7-4a9a-a475-bd645050c3fd"; // sub folder for faster tests
System.out.println("Clearing configuration.. ");
try {
engine.unsetSynchronizedFolder(folderId, false);
}catch(WorkspaceNotSynchedException e) {
}
System.out.println("Setting configuration");
SynchFolderConfiguration config=new SynchFolderConfiguration("public/netcdf/GPTest", "",
"f851ba11-bd3e-417a-b2c2-753b02bac506-98187548", // devVRE
"Agro",folderId);
engine.setSynchronizedFolder(config, folderId);
System.out.println("Invoke check... ");
engine.check(folderId, false);
System.out.println("Invoke synch");
engine.doSync(folderId);
Semaphore sem=new Semaphore(0);
engine.registerCallBack(folderId, new SyncOperationCallBack() {
@Override
public void onStep(ProcessStatus status, ProcessDescriptor descriptor) {
System.out.println("ON STEP : "+status+" "+descriptor);
System.out.println("LOG : \n"+ status.getLogBuilder().toString());
if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release();
}
});
System.out.println("Waiting for process.. ");
try {
sem.acquire();
} catch (InterruptedException e) {
}
System.out.println("Done");
engine.check(folderId, false);
// INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES
engine.shutDown();
}
}

67
src/test/java/org/gcube/usecases/ws/thredds/DTTests.java
Unescape Escape View File

@ -0,0 +1,67 @@
package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import org.apache.commons.io.IOUtils;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.data.transfer.model.PluginInvocation;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class DTTests {
public static void main(String[] args) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException, MalformedURLException, FileNotFoundException, IOException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
// TestCommons.setScope();
// String threddsHostname="thredds-d-d4s.d4science.org";
// DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+threddsHostname+":80");
// Destination toSetDestination=new Destination();
// toSetDestination.setCreateSubfolders(true);
// toSetDestination.setDestinationFileName("transferTest.tst");
// toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE);
// toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
// toSetDestination.setPersistenceId("thredds");
//
// //NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
// String fileLocation="WS-Tests/mySub";
// toSetDestination.setSubFolder(fileLocation);
//
// File temp=File.createTempFile("testTransfer", "tmp");
// IOUtils.copy(new URL("http://data-d.d4science.org/SUlDWjIxamdaUTdHcmpvdEFmcFFPOUcvbjF5VyswbXlHbWJQNStIS0N6Yz0").openStream(), new FileOutputStream(temp));
//
// System.out.println(client.localFile(temp,
// toSetDestination,Collections.singleton(new PluginInvocation(Constants.SIS_PLUGIN_ID))));
//
TokenSetter.set("/gcube/devNext");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
String folderId=ws.getItemByPath("/Workspace/Accounting").getId();
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId);
System.out.println("Done");
}
}

78
src/test/java/org/gcube/usecases/ws/thredds/FileAccessTests.java
Unescape Escape View File

@ -0,0 +1,78 @@
package org.gcube.usecases.ws.thredds;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.UnableToLockException;
public class FileAccessTests {
public static void main(String[] args) throws InternalException {
TestCommons.setScope();
String processID="stupidProcess";
String toLockFolder="devVRE";
final ThreddsController controller=new ThreddsController(toLockFolder, TokenSetter.getCurrentToken());
Runnable readCatalog=new Runnable() {@Override public void run() { try{
controller.readThreddsFile("catalog.xml");
}catch(Throwable t) {System.err.println(t); }}};
Runnable readInfo=new Runnable() {@Override public void run() { try{
controller.getThreddsInfo();
}catch(Throwable t) {System.err.println(t); }}};
repeatedTimedOperation(readCatalog,"READ CATALOG");
repeatedTimedOperation(readInfo, "READ INFO");
System.out.println("Checking lock ..");
try {
controller.lockFolder(processID);
}catch(UnableToLockException e) {
System.out.println("Dirty from previous tests.. removing lock..");
controller.deleteThreddsFile(Constants.LOCK_FILE);
System.out.println("Trying to lock again..");
controller.lockFolder(processID);
}
try{
System.out.println("Reading file descriptor... ");
System.out.println(controller.getFileDescriptor(Constants.LOCK_FILE));
System.out.println("Ok, next should fail...");
controller.lockFolder(processID);
}catch(UnableToLockException e) {
System.out.println("Correctly failed, now we read lock.. ");
String lockId=controller.readThreddsFile(Constants.LOCK_FILE);
if(lockId.equals(processID)) {
System.out.println("Lock owned. Going to remove it.. ");
controller.deleteThreddsFile(Constants.LOCK_FILE);
System.out.println("Now I can lock it again... ");
controller.lockFolder(processID);
System.out.println("Ok, cleaning it up.. ");
controller.deleteThreddsFile(Constants.LOCK_FILE);
}else throw new RuntimeException("LOCK ID "+lockId+" IS DIFFERENT then expected "+processID);
}
}
private static void repeatedTimedOperation(Runnable op,String title) {
System.out.println(title);
for(int i=0;i<10;i++) {
long startTime=System.currentTimeMillis();
op.run();
System.out.println("["+i+"] in "+(System.currentTimeMillis()-startTime));
}
}
}

46
src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java
Unescape Escape View File

@ -0,0 +1,46 @@
package org.gcube.usecases.ws.thredds;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.homelibrary.home.workspace.folder.FolderItem;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class GetWSSynchSpace {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
TokenSetter.set("/gcube/preprod/preVRE");
SyncEngine engine=SyncEngine.get();
String folderId=TestCommons.getWSIdByPath("/Workspace/Thredds main catalog");
System.out.println(folderId);
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
WorkspaceFolder folder=(WorkspaceFolder) ws.getItem(folderId);
SynchFolderConfiguration config=new SynchFolderConfiguration();
System.out.println("Total size : "+computeLength(folder, config));
}
private static final long computeLength(WorkspaceItem item,SynchFolderConfiguration config) throws InternalErrorException {
long toReturn=0l;
if(item.isFolder()) {
for(WorkspaceItem child:((WorkspaceFolder)item).getChildren()) {
if(item.isFolder()||config.matchesFilter(item.getName()))
toReturn=toReturn+computeLength(child, config);
}
}else toReturn=toReturn+((FolderItem)item).getLength();
return toReturn;
}
}

103
src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java
Unescape Escape View File

@ -0,0 +1,103 @@
package org.gcube.usecases.ws.thredds;
import java.io.File;
import java.util.Map;
import java.util.Map.Entry;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Properties;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
import org.gcube.data.transfer.library.faults.FailedTransferException;
import org.gcube.data.transfer.library.faults.InitializationException;
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
import org.gcube.data.transfer.library.faults.InvalidSourceException;
import org.gcube.data.transfer.library.faults.SourceNotSetException;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.DestinationClashPolicy;
import org.gcube.spatial.data.sdi.interfaces.Metadata;
import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class PublicLinkIssueTest {
public static void main (String[] args) throws ItemNotFoundException, InternalErrorException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
TokenSetter.set("/d4science.research-infrastructures.eu");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
System.out.println(ws.getItemByPath("/Workspace/ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc").getPublicLink(false));
String threddsHostName = Commons.getThreddsHost();
DataTransferClient client=Commons.getDTClient(threddsHostName);
Destination dest=new Destination("thredds", "public/netcdf/someWhere", "myTest.txt", true, DestinationClashPolicy.REWRITE, DestinationClashPolicy.APPEND);
client.httpSource("http://data.d4science.org/V2drR2gxSFRTQlpLVC9nakozL29QcDdPR2U5UEVHYWRHbWJQNStIS0N6Yz0", dest);
scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d"));
System.out.println("OK FIRST ...");
try {
Metadata meta=SDIAbstractPlugin.metadata().build();
MetadataPublishOptions opts=new MetadataPublishOptions(new TemplateInvocationBuilder().threddsOnlineResources(threddsHostName, "myMeta", "testCatalog").get());
opts.setGeonetworkCategory("Datasets");
MetadataReport report=meta.pushMetadata(new File("/home/fabio/Desktop/meta.xml"), opts);
}catch(Exception e) {
e.printStackTrace();
}
System.out.println("CHECKING AGAIN");
scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d"));
}
public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException {
System.out.println("Folder "+folder.getPath());
printProperties(folder.getProperties());
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
for(WorkspaceItem item:folder.getChildren())
if(!item.isFolder()&&config.matchesFilter(item.getName())) {
// System.out.println("ITEM "+item.getPath());
printProperties(item.getProperties());
}
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder())scanForPrint((WorkspaceFolder) item);
}
public static void printProperties(Properties prop) throws InternalErrorException {
Map<String,String> map=prop.getProperties();
// System.out.print("Properties : ..");
for(Entry<String,String> entry:map.entrySet()) {
if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {
// if(true) {
if(entry.getValue()==null) System.out.print(entry.getKey()+" is null;");
else System.out.print(entry.getKey()+" = "+entry.getValue()+";");
}
}
// System.out.println();
}
}

113
src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java
Unescape Escape View File

@ -0,0 +1,113 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Map;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
import lombok.AllArgsConstructor;
import lombok.Getter;
public class TestCommons {
@Getter
@AllArgsConstructor
static class TestSet{
String label;
String scope;
String folderId;
String remotePath;
String targetToken;
String toCreateCatalogName;
public String getFolderId() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
if(folderId.startsWith("/"))
return getWSIdByPath(folderId);
else return folderId;
}
}
private static Map<String,TestSet> configs=new HashMap<>();
private static String toUseConfig="pre";
static {
configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","",""));
// folderName="WS-Tests";
configs.put("default", new TestSet("Default Tests","/gcube/devsec/devVRE","/Workspace/ThreddsDev","public/netcdf","f851ba11-bd3e-417a-b2c2-753b02bac506-98187548","main"));
configs.put("pre", new TestSet("Default Tests","/gcube/preprod/preVRE","/Workspace/CMEMS","public/netcdf/CMEMS","97cfc53e-7f71-4676-b5e0-bdd149c8460f-98187548","main"));
}
public static String getWSIdByPath(String path) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException {
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
return ws.getItemByPath(path).getId();
}
public static void setScope() {
TokenSetter.set(configs.get(toUseConfig).getScope());
}
public static WorkspaceFolder getTestFolder() throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, IOException {
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
try{
return (WorkspaceFolder) ws.getItem(configs.get(toUseConfig).getFolderId());
}catch(Throwable t) {
// try to use path
return (WorkspaceFolder) ws.getItemByPath(configs.get(toUseConfig).getFolderId());
}
// WorkspaceFolder folder=null;
// try{
// folder=ws.getRoot().createFolder(folderName+"2", "test purposes");
// }catch(ClassCastException e) {
// folder=(WorkspaceFolder) ws.getItemByPath("/Workspace/"+folderName+"2");
// }
//
// String datasetUrl="https://thredds-d-d4s.d4science.org/thredds/fileServer/public/netcdf/test%20by%20Francesco/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc";
// try {
// folder.createExternalFileItem("dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc", "nc test file", "application/x-netcdf", new URL(datasetUrl).openStream());
// }catch(Exception e) {
// // file already existing..
// }
// return folder;
}
public static ThreddsController getThreddsController() throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
SynchFolderConfiguration config=getSynchConfig();
return new ThreddsController(config.getRemotePath(), config.getTargetToken());
}
public static SynchFolderConfiguration getSynchConfig() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
TestSet set=configs.get(toUseConfig);
return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId());
}
}

71
src/test/java/org/gcube/usecases/ws/thredds/ThreddsCatalogTests.java
Unescape Escape View File

@ -0,0 +1,71 @@
package org.gcube.usecases.ws.thredds;
import org.gcube.data.transfer.library.utils.ScopeUtils;
import org.gcube.data.transfer.model.plugins.thredds.DataSetScan;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.faults.InternalException;
public class ThreddsCatalogTests {
public static void main(String[] args) throws InternalException {
TokenSetter.set("/gcube/devsec");
System.out.println("Current scope is "+ScopeUtils.getCurrentScope());
// SyncEngine engine=SyncEngine.get();
//
// String[] toCheckTokens=new String[] {
// "123da564-2af2-4023-90f5-01a934d80754-98187548",
// "678abd6f-1a39-4e52-9c14-9288bf28a2ed-98187548",
// "f851ba11-bd3e-417a-b2c2-753b02bac506-98187548",
// "1dd10a45-4b04-4fba-b878-ed6e40d235db-843339462",
// "feda0617-cd9d-4841-b6f0-e047da5d32ed-98187548",
// "5741e3e4-dbde-46fa-828d-88da609e0517-98187548",
// "97cfc53e-7f71-4676-b5e0-bdd149c8460f-98187548",
// "6a16458f-d514-4c83-b012-6f2b6bf19794-843339462",
// "84bcb500-100e-4d35-868d-d6f3dbb95ade-843339462",
// "9dd56598-4092-460d-bfe4-91ecff66290a-843339462",
// "adb3f8f5-bf55-4d31-b951-c60139ff8b85-843339462",
// "39f574c0-d439-4c29-a652-01d97472a4fb-843339462",
// "3701435c-1c36-494f-ae31-4d002910e27e-843339462",
// "cc491b9c-c75b-41a8-a999-4c9f6f25263d-843339462",
// "74673327-645e-4b0b-8382-2edd663f3a38-843339462",
// "65f3fc8b-1bcc-4ae7-a71f-97e671a27cb7-843339462",
// "3240d5ec-72e6-4e03-b0c7-eaa24d91ea80-843339462",
// };
//
//
// for(String token:toCheckTokens) {
// System.out.println("Checking catalog for token "+token);
// for(CatalogBean bean:engine.getAvailableCatalogsByToken(token))
// System.out.println(bean.getName()+" in "+bean.getPath()+" Default : "+bean.getIsDefault());
//
// System.out.println("************************************");
// System.out.println();
// }
//
ThreddsController controller=new ThreddsController("",TokenSetter.getCurrentToken());
ThreddsInfo info=controller.getThreddsInfo();
String [] paths=new String[] {
"/data/content/thredds/newer",
"/data/content/thredds/public",
"/data/content/thredds/public/netcdf",
"/data/content/thredds/public/netcdf/GPTest",
};
for(String path:paths) {
if(info.getCatalogByFittingLocation(path)!=null) {
DataSetScan ds=(DataSetScan) info.getDataSetFromLocation(path);
if(ds==null)
System.out.println("Catalog for "+path+"\t is null.");
else System.out.println("Catalog for "+path+"\t : "+ds.getLocation()+"\t "+ds.getName());
}else System.out.println("No catalog for path "+path);
}
}
}

30
src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java
Unescape Escape View File

@ -0,0 +1,30 @@
package org.gcube.usecases.ws.thredds;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class ThreddsTests {
public static void main(String[] args) throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException {
TestCommons.setScope();
SynchFolderConfiguration folderConfig=TestCommons.getSynchConfig();
ThreddsController controller=TestCommons.getThreddsController();
System.out.println("Getting thredds info...");
ThreddsInfo info=controller.getThreddsInfo();
System.out.println("INFO "+info);
System.out.println(info.getCatalogByFittingLocation(info.getLocalBasePath()+"/"+folderConfig.getRemotePath()));
}
}

40
src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java
Unescape Escape View File

@ -0,0 +1,40 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
public class WSTimes {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, InternalErrorException, HomeNotFoundException, UserNotFoundException, IOException {
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
for(WorkspaceItem item : folder.getChildren()) {
printDates(item);
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
}
item.getProperties().addProperty("Fake prop", "fake value");
printDates(item);
}
}
public static final void printDates(WorkspaceItem item) throws InternalErrorException {
System.out.println("ITEM : "+item.getName());
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getCreationTime().getTime()));
System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getLastModificationTime().getTime()));
}
}

75
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java
Unescape Escape View File

@ -0,0 +1,75 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.util.Date;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
public class WorkspaceAccounting {
public static void main(String[] args) throws InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, IOException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, NumberFormatException, ItemNotFoundException {
TestCommons.setScope();
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
// WorkspaceFolder folder=ws.getRoot().createFolder("Accounting", "test purposes");
//
//
// WorkspaceFolder subFolder=folder.createFolder("SubFolder", "Will be removed");
//
//
// ExternalFile toBeRemovedFile=folder.createExternalFileItem("The file", "file to be removed", "application/xml", File.createTempFile("tmp", ".tmp"));
//
// subFolder.remove();
// toBeRemovedFile.remove();
//
//
// for(AccountingEntry entry:folder.getAccounting()) {
// try {
//
//
// Date eventTime=entry.getDate().getTime();
// String toDeleteRemote=null;
// switch(entry.getEntryType()) {
// case REMOVAL:{
// AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry;
// System.out.println(removalEntry.getItemName() +"REMOVED. FolderItemType "+removalEntry.getFolderItemType()+" ItemType "+removalEntry.getItemType());
//
//
// break;
// }
// case RENAMING:{
// AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry;
// toDeleteRemote=renamingEntry.getOldItemName();
//
// }
// case CUT:{
// AccountingEntryCut cut = (AccountingEntryCut) entry;
// toDeleteRemote=cut.getItemName();
// break;
// }
// }
//
// }catch(Throwable t) {
// t.printStackTrace();
// }
// }
//
// }
System.out.println(WorkspaceUtils.isModifiedAfter(ws.getItemByPath(TestCommons.getTestFolder().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc"), new Date(0l)));
}
}

28
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java
Unescape Escape View File

@ -0,0 +1,28 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
public class WorkspaceCleanup {
public static void main(String[] args) throws WorkspaceInteractionException, InternalException, InternalErrorException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, HomeNotFoundException, UserNotFoundException, IOException {
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
SyncEngine engine=SyncEngine.get();
engine.forceUnlock(folder.getId());
engine.unsetSynchronizedFolder(folder.getId(), false);
}
}

62
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java
Unescape Escape View File

@ -0,0 +1,62 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Collections;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceLock {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, HomeNotFoundException, UserNotFoundException, WorkspaceInteractionException, InternalException, MalformedURLException, IOException {
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
WorkspaceFolderManager manager=new WorkspaceFolderManager(folder.getId());
String processID="mytest";
try {
System.out.println("Trying to cleanup, first.. ");
folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null));
System.out.println("FOLDER PROPERTIES : "+folder.getProperties().getProperties());
}catch(Throwable t) {
System.err.println("Dismiss error : ");
t.printStackTrace();
}
// manager.configure(new SynchFolderConfiguration("mySynchedCatalog","*.nc,*.ncml,*.asc",SecurityTokenProvider.instance.get()));
System.out.println("Is locked : "+manager.isLocked());
System.out.println("locking ... ");
manager.lock(processID);
try {
manager.dismiss(false);
System.err.println("It should have raised locked exception");
}catch(Exception e){
System.out.println("Ok check lock on dismiss ");
}
manager.unlock(processID);
manager.dismiss(false);
System.out.println("This should be false : "+manager.isSynched());
}
}

93
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java
Unescape Escape View File

@ -0,0 +1,93 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Map;
import java.util.Map.Entry;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Properties;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceProperties {
public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, ItemNotFoundException, MalformedURLException, IOException, WorkspaceInteractionException, InternalException {
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
SyncEngine.get().check(folder.getId(), true);
scanForPrint(folder);
// SyncEngine.get().shutDown();
// for(Workspace)
// printProperties(folder.getProperties());
//
// System.out.println("Has property : "+folder.getProperties().hasProperty(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS));
// printProperties(folder.getProperties());
//
//
// System.out.println("Setting property.. ");
// folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, "true"));
//
// System.out.println("Has property : "+folder.getProperties().hasProperty(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS));
// printProperties(folder.getProperties());
//
//
// System.out.println("Removing (setting it null) ");
// folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null));
//
// System.out.println("Has property : "+folder.getProperties().hasProperty(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS));
// printProperties(folder.getProperties());
//
// String folderId=
//
// WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId);
//
// manager.configure(new SynchFolderConfiguration("myRemoteFolder","thredds","*.nc,*.ncml,*.asc"));
//
// manager.dismiss(false);
}
public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException {
System.out.println("Folder "+folder.getPath());
printProperties(folder.getProperties());
SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "","");
for(WorkspaceItem item:folder.getChildren())
if(!item.isFolder()&&config.matchesFilter(item.getName())) {
System.out.println("ITEM "+item.getPath());
printProperties(item.getProperties());
}
for(WorkspaceItem item:folder.getChildren())
if(item.isFolder())scanForPrint((WorkspaceFolder) item);
}
public static void printProperties(Properties prop) throws InternalErrorException {
Map<String,String> map=prop.getProperties();
System.out.print("Properties : ..");
for(Entry<String,String> entry:map.entrySet()) {
// if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)||
// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) {
if(true) {
if(entry.getValue()==null) System.out.print(entry.getKey()+" is null;");
else System.out.print(entry.getKey()+" = "+entry.getValue()+";");
}
}
System.out.println();
}
}

104
src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java
Unescape Escape View File

@ -0,0 +1,104 @@
package org.gcube.usecases.ws.thredds;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.concurrent.Semaphore;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
import org.gcube.usecases.ws.thredds.faults.InternalException;
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
public class WorkspaceSynchronizationTest {
public static void main(String[] args) throws ProcessNotFoundException, InternalErrorException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, HomeNotFoundException, UserNotFoundException, MalformedURLException, IOException {
// GET ENGINE : SINGLETON INSTANCE
SyncEngine engine=SyncEngine.get();
engine.setRequestLogger("requests.txt");
//TEST INFO...
TestCommons.setScope();
WorkspaceFolder folder=TestCommons.getTestFolder();
// FOLDER CONFIGURATION BEAN
SynchFolderConfiguration config=TestCommons.getSynchConfig();
// try {
// //try to clean it up, first..
// System.out.println("Cleaning it up..");
// engine.unsetSynchronizedFolder(folder.getId(), false);
// }catch(WorkspaceNotSynchedException e) {
// // it was already cleared
// }catch(WorkspaceLockedException e) {
// engine.forceUnlock(folder.getId());
// engine.unsetSynchronizedFolder(folder.getId(), false);
// }
try {
// WHEN OPENING A FOLDER, INVOKE CHECK TO UPDATE SYNCH STATUS
engine.check(folder.getId(), false);
}catch(WorkspaceNotSynchedException e) {
System.out.println("Folder not synched, configurin it..");
engine.setSynchronizedFolder(config, folder.getId());
engine.check(folder.getId(), false);
}catch(WorkspaceLockedException e) {
System.out.println("Workspace locked, going to force unlock.."); // MAINLY FOR TEST PURPOSES, OR WHEN SOMETHIGN GOES WRONG.. USE CAUTIOUSLY
engine.forceUnlock(folder.getId());
engine.check(folder.getId(), false);
}
// INVOKE SYNCHRONIZATION ON FOLDER
ProcessDescriptor descriptor=engine.doSync(folder.getId());
System.out.println("Obtained descriptor : "+descriptor);
Semaphore sem=new Semaphore(0);
// REGISTER CALLBACK TO MONITOR PROGRESS
engine.registerCallBack(folder.getId(), new SyncOperationCallBack() {
@Override
public void onStep(ProcessStatus status, ProcessDescriptor descriptor) {
System.out.println("ON STEP : "+status+" "+descriptor);
System.out.println("LOG : \n"+ status.getLogBuilder().toString());
if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release();
}
});
System.out.println("Waiting for process.. ");
try {
sem.acquire();
} catch (InterruptedException e) {
}
engine.check(folder.getId(), true);
// INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES
engine.shutDown();
}
}
Write Preview
Loading…
Cancel
Save