From 98e50a055e15ae2ac75f236578bfaf2f158a711e Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Fri, 18 May 2018 15:38:19 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/spatial-data/ws-thredds@167619 82a268e6-3cf1-43bd-a215-b396298e98cf --- .classpath | 36 ++ .project | 23 ++ .settings/org.eclipse.core.resources.prefs | 6 + .settings/org.eclipse.jdt.core.prefs | 5 + .settings/org.eclipse.m2e.core.prefs | 4 + distro/LICENSE | 1 + distro/README | 65 ++++ distro/changelog.xml | 8 + distro/descriptor.xml | 30 ++ distro/profile.xml | 29 ++ pom.xml | 69 ++++ requests.txt | 0 .../gcube/usecases/ws/thredds/Commons.java | 162 ++++++++ .../gcube/usecases/ws/thredds/Constants.java | 94 +++++ .../ws/thredds/FolderConfiguration.java | 40 ++ .../usecases/ws/thredds/FolderReport.java | 31 ++ .../ws/thredds/LocalConfiguration.java | 44 +++ .../gcube/usecases/ws/thredds/NetUtils.java | 57 +++ .../usecases/ws/thredds/PublishFolders.java | 283 ++++++++++++++ .../gcube/usecases/ws/thredds/SyncEngine.java | 57 +++ .../usecases/ws/thredds/TokenSetter.java | 55 +++ .../ws/thredds/engine/PublishReport.java | 22 ++ .../ws/thredds/engine/PublishRequest.java | 64 ++++ .../ws/thredds/engine/PublishThread.java | 217 +++++++++++ .../thredds/engine/TransferRequestServer.java | 102 +++++ .../ws/thredds/engine/impl/Process.java | 312 +++++++++++++++ .../engine/impl/ProcessDescriptor.java | 23 ++ .../ws/thredds/engine/impl/ProcessStatus.java | 50 +++ .../thredds/engine/impl/SynchEngineImpl.java | 267 +++++++++++++ .../engine/impl/ThreddsController.java | 318 ++++++++++++++++ .../engine/impl/WorkspaceFolderManager.java | 351 +++++++++++++++++ .../thredds/engine/impl/WorkspaceUtils.java | 356 ++++++++++++++++++ .../impl/threads/DeleteRemoteRequest.java | 18 + .../impl/threads/ProcessIdProvider.java | 30 ++ .../threads/ProcessInitializationThread.java | 59 +++ .../engine/impl/threads/RequestLogger.java | 113 ++++++ .../impl/threads/SynchronizationRequest.java | 16 + .../impl/threads/SynchronizationThread.java | 289 ++++++++++++++ .../threads/TransferFromThreddsRequest.java | 26 ++ .../threads/TransferToThreddsRequest.java | 23 ++ .../thredds/faults/CancellationException.java | 35 ++ .../faults/DataTransferPluginError.java | 35 ++ .../ws/thredds/faults/InternalException.java | 37 ++ .../thredds/faults/LockNotOwnedException.java | 35 ++ .../faults/ProcessNotFoundException.java | 35 ++ .../faults/RemoteFileNotFoundException.java | 35 ++ .../thredds/faults/UnableToLockException.java | 35 ++ .../WorkspaceFolderNotRootException.java | 35 ++ .../faults/WorkspaceInteractionException.java | 37 ++ .../faults/WorkspaceLockedException.java | 35 ++ .../faults/WorkspaceNotSynchedException.java | 35 ++ .../ws/thredds/model/CompletionCallback.java | 9 + .../usecases/ws/thredds/model/StepReport.java | 25 ++ .../thredds/model/SyncFolderDescriptor.java | 26 ++ .../thredds/model/SyncOperationCallBack.java | 10 + .../ws/thredds/model/SyncOperationTicket.java | 5 + .../model/SynchFolderConfiguration.java | 37 ++ .../model/SynchronizedElementInfo.java | 10 + .../ws/thredds/model/gui/CatalogBean.java | 43 +++ src/main/resources/log4j.properties | 11 + src/main/resources/logback.xml | 0 .../ws/thredds/configuration.properties | 6 + .../usecases/ws/thredds/DTSynchUseCase.java | 78 ++++ .../gcube/usecases/ws/thredds/DTTests.java | 67 ++++ .../usecases/ws/thredds/FileAccessTests.java | 78 ++++ .../usecases/ws/thredds/GetWSSynchSpace.java | 46 +++ .../ws/thredds/PublicLinkIssueTest.java | 103 +++++ .../usecases/ws/thredds/TestCommons.java | 113 ++++++ .../ws/thredds/ThreddsCatalogTests.java | 71 ++++ .../usecases/ws/thredds/ThreddsTests.java | 30 ++ .../gcube/usecases/ws/thredds/WSTimes.java | 40 ++ .../ws/thredds/WorkspaceAccounting.java | 75 ++++ .../usecases/ws/thredds/WorkspaceCleanup.java | 28 ++ .../usecases/ws/thredds/WorkspaceLock.java | 62 +++ .../ws/thredds/WorkspaceProperties.java | 93 +++++ .../thredds/WorkspaceSynchronizationTest.java | 104 +++++ 76 files changed, 5314 insertions(+) create mode 100644 .classpath create mode 100644 .project create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 .settings/org.eclipse.jdt.core.prefs create mode 100644 .settings/org.eclipse.m2e.core.prefs create mode 100644 distro/LICENSE create mode 100644 distro/README create mode 100644 distro/changelog.xml create mode 100644 distro/descriptor.xml create mode 100644 distro/profile.xml create mode 100644 pom.xml create mode 100644 requests.txt create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/Commons.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/Constants.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/FolderConfiguration.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/FolderReport.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/TokenSetter.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/PublishReport.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/TransferRequestServer.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessDescriptor.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessStatus.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessIdProvider.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/CancellationException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/DataTransferPluginError.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/InternalException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/LockNotOwnedException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/ProcessNotFoundException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/RemoteFileNotFoundException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/UnableToLockException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceFolderNotRootException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceInteractionException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceLockedException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceNotSynchedException.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/CompletionCallback.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/StepReport.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/SyncFolderDescriptor.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationCallBack.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationTicket.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/SynchFolderConfiguration.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/SynchronizedElementInfo.java create mode 100644 src/main/java/org/gcube/usecases/ws/thredds/model/gui/CatalogBean.java create mode 100644 src/main/resources/log4j.properties create mode 100644 src/main/resources/logback.xml create mode 100644 src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/DTSynchUseCase.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/DTTests.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/FileAccessTests.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/ThreddsCatalogTests.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java create mode 100644 src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..fae1a2b --- /dev/null +++ b/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..ff54a5a --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + ws-thredds + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..714351a --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..ff7698f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=false +version=1 diff --git a/distro/LICENSE b/distro/LICENSE new file mode 100644 index 0000000..2d9616a --- /dev/null +++ b/distro/LICENSE @@ -0,0 +1 @@ +${gcube.license} \ No newline at end of file diff --git a/distro/README b/distro/README new file mode 100644 index 0000000..9ac47a3 --- /dev/null +++ b/distro/README @@ -0,0 +1,65 @@ +The gCube System - ${name} +-------------------------------------------------- + +${description} + + +${gcube.description} + +${gcube.funding} + + +Version +-------------------------------------------------- + +${version} (${buildDate}) + +Please see the file named "changelog.xml" in this directory for the release notes. + + + +Authors +-------------------------------------------------- + +* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). + +Maintainers +----------- + +* Fabio Sinibaldi (fabio.sinibaldi-AT-isti.cnr.it) Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" - CNR, Pisa (Italy). + + +Download information +-------------------------------------------------- + +Source code is available from SVN: + ${scm.url} + +Binaries can be downloaded from the gCube website: + ${gcube.website} + +Installation +-------------------------------------------------- + +Installation documentation is available on-line in the gCube Wiki: + ${gcube.wikiRoot}/Tabular_Data_Manager + +Documentation +-------------------------------------------------- + +Documentation is available on-line in the gCube Wiki: + ${gcube.wikiRoot}/Tabular_Data_Manager + ${gcube.wikiRoot}/Tabular_Data_Manager + + +Support +-------------------------------------------------- + +Bugs and support requests can be reported in the gCube issue tracking tool: + ${gcube.issueTracking} + + +Licensing +-------------------------------------------------- + +This software is licensed under the terms you may find in the file named "LICENSE" in this directory. \ No newline at end of file diff --git a/distro/changelog.xml b/distro/changelog.xml new file mode 100644 index 0000000..b2f8291 --- /dev/null +++ b/distro/changelog.xml @@ -0,0 +1,8 @@ + + + First Release + + + Cleanup folder fix + + \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml new file mode 100644 index 0000000..dc46ade --- /dev/null +++ b/distro/descriptor.xml @@ -0,0 +1,30 @@ + + servicearchive + + tar.gz + + / + + + ${distroDirectory} + / + true + + README + LICENSE + changelog.xml + + 755 + true + + + + + target/${build.finalName}.${project.packaging} + /${artifactId} + + + \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml new file mode 100644 index 0000000..703fb88 --- /dev/null +++ b/distro/profile.xml @@ -0,0 +1,29 @@ + + + + Service + + ${description} + Portlets + ${artifactId} + 1.0.0 + + + ${description} + ${artifactId} + ${version} + + ${groupId} + ${artifactId} + ${version} + + library + + ${build.finalName}.jar + + + + + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..f728b39 --- /dev/null +++ b/pom.xml @@ -0,0 +1,69 @@ + + 4.0.0 + + org.gcube.tools + maven-parent + LATEST + + org.gcube.spatial-data + ws-thredds + 0.2.0-SNAPSHOT + ws-thredds + prototype of WS integration with data-transfer for Thredds pubblication + + + ${project.basedir}/distro + http://svn.research-infrastructures.eu/d4science/gcube/trunk/portlets/user/${project.artifactId} + + + + + scm:svn:${svnBaseUrl}/${project.artifactId} + scm:svn:${svnBaseUrl}/${project.artifactId} + ${svnBaseUrl}/${project.artifactId} + + + + + + + + org.gcube.spatial.data + sdi-library + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + org.gcube.data.transfer + data-transfer-library + [1.2.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + + + + + org.gcube.common + home-library-jcr + [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) + + + + org.gcube.common + home-library + [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) + + + + org.slf4j + slf4j-log4j12 + 1.6.4 + provided + + + + + + \ No newline at end of file diff --git a/requests.txt b/requests.txt new file mode 100644 index 0000000..e69de29 diff --git a/src/main/java/org/gcube/usecases/ws/thredds/Commons.java b/src/main/java/org/gcube/usecases/ws/thredds/Commons.java new file mode 100644 index 0000000..28dcbb9 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/Commons.java @@ -0,0 +1,162 @@ +package org.gcube.usecases.ws.thredds; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.text.SimpleDateFormat; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; + +import org.apache.tika.io.IOUtils; +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.client.AuthorizationFilter; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; +import org.gcube.usecases.ws.thredds.faults.UnableToLockException; +import org.glassfish.jersey.client.ClientConfig; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Commons { + + + + static final SimpleDateFormat DATE_FORMAT= new SimpleDateFormat("dd-MM-yy:HH:mm:SS"); + + + public static void cleanupFolder(String toCleanPath, String destinationToken) { + String toRestoreToken=TokenSetter.getCurrentToken(); + try{ + log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath); + TokenSetter.setToken(destinationToken); + String hostname=getThreddsHost(); + DataTransferClient client=getDTClient(hostname); + File toTransfer=File.createTempFile("clean", ".dt_temp"); + toTransfer.createNewFile(); + Destination dest=new Destination(); + dest.setCreateSubfolders(true); + dest.setOnExistingFileName(DestinationClashPolicy.REWRITE); + dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE); + dest.setPersistenceId("thredds"); + dest.setSubFolder("public/netcdf/"+toCleanPath); + log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname); + client.localFile(toTransfer, dest); + log.info("Done"); + }catch(Exception e) { + log.error("Unable to delete remote folder "+toCleanPath,e); + throw new RuntimeException("Unable to cleanup remote folder."); + }finally { + log.debug("Resetting original token {} ",toRestoreToken); + TokenSetter.set(toRestoreToken); + } + } + + + public static String getThreddsHost(){ + + SimpleQuery query =queryFor(GCoreEndpoint.class); + query.addCondition("$resource/Profile/ServiceClass/text() eq 'SDI'") + .addCondition("$resource/Profile/ServiceName/text() eq 'Thredds'"); + // .setResult("$resource/Profile/AccessPoint"); + + DiscoveryClient client = clientFor(GCoreEndpoint.class); + + GCoreEndpoint endpoint= client.submit(query).get(0); + + return endpoint.profile().endpoints().iterator().next().uri().getHost(); + } + + public static DataTransferClient getDTClient(String threddsHostName) throws UnreachableNodeException, ServiceNotFoundException { + log.debug("Getting DT Client for {} ",threddsHostName); + return DataTransferClient.getInstanceByEndpoint("http://"+threddsHostName+":80"); + } + + public static String readThreddsFile(String location) throws RemoteFileNotFoundException { + String urlString="http://"+getThreddsHost()+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+location; + log.info("Reading file at {} ",urlString); + try{ + return getWebClient().target(urlString).request().get().readEntity(String.class); + }catch(Throwable t) { + throw new RemoteFileNotFoundException("Unable to access "+urlString, t); + } + } + + + + public static void deleteThreddsFile(String location) throws RemoteFileNotFoundException { + String urlString="http://"+getThreddsHost()+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+location; + log.info("Reading file at {} ",urlString); + try{ + getWebClient().target(urlString).request().delete(); + }catch(Throwable t) { + throw new RemoteFileNotFoundException("Unable to access "+urlString, t); + } + } + + public static final ThreddsInfo getThreddsInfo() { + String infoPath=getThreddsInfoPath(); + log.info("Loading thredds info from {} ",infoPath); + WebTarget target=getWebClient().target(infoPath); + return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class); + } + + private static Client getWebClient() { + return ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class)); + } + + private static String getThreddsInfoPath() { + return "https://"+getThreddsHost()+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG"; + } + + public static void lockFolder(String folderPath,String processId) throws UnableToLockException { + PrintWriter writer=null; + File temp=null; + try{ + log.info("Locking remote path {} to processId {} ",folderPath,processId); + DataTransferClient cl=getDTClient(getThreddsHost()); + + Destination dest=new Destination(); + dest.setCreateSubfolders(false); + dest.setOnExistingFileName(DestinationClashPolicy.FAIL); + dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + dest.setPersistenceId("thredds"); + dest.setSubFolder(folderPath); + dest.setDestinationFileName(Constants.LOCK_FILE); + + temp=File.createTempFile("tmp_lock", ".tmp"); + writer=new PrintWriter(temp); + writer.write(processId); + writer.flush(); + writer.close(); + + cl.localFile(temp, dest); + }catch(Throwable t) { + throw new UnableToLockException("Unable to lock "+folderPath,t); + }finally { + if(writer!=null) IOUtils.closeQuietly(writer); + if(temp!=null)try { + Files.deleteIfExists(temp.toPath()); + }catch(IOException e) { + log.warn("Unable to delete temp file {} ",temp.getAbsolutePath(),e); + } + } + + } + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/Constants.java b/src/main/java/org/gcube/usecases/ws/thredds/Constants.java new file mode 100644 index 0000000..3b81177 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/Constants.java @@ -0,0 +1,94 @@ +package org.gcube.usecases.ws.thredds; + +import java.text.SimpleDateFormat; +import java.util.HashMap; +import java.util.Map; + +public class Constants { + + public static final SimpleDateFormat DATE_FORMAT= new SimpleDateFormat("dd-MM-yy:HH:mm:SS"); + + public static final String LOCK_FILE="~WS-LOCK.lock"; + public static final String GCUBE_TOKEN_HEADER="gcube-token"; + + public static final String LAST_UPDATE_TIME="WS-SYNCH.LAST_UPDATE"; + + public static final String THREDDS_PERSISTENCE="thredds"; + public static final String THREDDS_DATA_TRANSFER_BASE_URL="data-transfer-service/gcube/service/REST/"+THREDDS_PERSISTENCE+"/"; + + + public static final String SDI_THREDDS_BASE_URL="sdi-service/gcube/service/Thredds"; + + + public static final String SIS_PLUGIN_ID="SIS/GEOTK"; + + public static class WorkspaceProperties{ + // Folder + public static final String SYNCH_FILTER="WS-SYNCH.SYNCHRONIZATION-FILTER"; + public static final String TARGET_TOKEN="WS-SYNCH.TARGET-TOKEN"; + public static final String REMOTE_PATH="WS-SYNCH.REMOTE-PATH"; + public static final String REMOTE_PERSISTENCE="WS-SYNCH.REMOTE-PERSISTENCE"; + public static final String RELATED_CATALOG="WS-SYNCH.RELATED-CATALOG"; + public static final String VALIDATE_METADATA="WS-SYNCH.VALIDATE-METADATA"; + public static final String ROOT_FOLDER_ID="WS-SYNCH.ROOT-FOLDER-ID"; + + + // Common + public static final String TBS="WS-SYNCH.TO-BE-SYNCHRONIZED"; + public static final String SYNCHRONIZATION_STATUS="WS-SYNCH.SYNCH-STATUS"; + public static final String LAST_UPDATE_TIME="WS-SYNCH.LAST-UPDATE-TIME"; + public static final String LAST_UPDATE_STATUS="WS-SYNCH.LAST-UPDATE-STATUS"; + + // ITEM + public static final String METADATA_UUID="WS-SYNCH.METADATA-UUID"; + } + + + + public static class Configuration{ + public static final String SCANNER_POOL_MAX_SIZE="scanner.pool.maxSize"; + public static final String SCANNER_POOL_CORE_SIZE="scanner.pool.coreSize"; + public static final String SCANNER_POOL_IDLE_MS="scanner.pool.idle.ms"; + + public static final String TRANSFERS_POOL_MAX_SIZE="transfers.pool.maxSize"; + public static final String TRANSFERS_POOL_CORE_SIZE="transfers.pool.coreSize"; + public static final String TRANSFERS_POOL_IDLE_MS="transfers.pool.idle.ms"; + } + + public static final Map cleanedItemPropertiesMap=new HashMap(); + + public static final Map cleanedFolderPropertiesMap=new HashMap(); + + public static final Map defaultConfigurationMap=new HashMap(); + + static { + cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.TBS, null); + cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, null); + cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, null); + cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, null); + cleanedItemPropertiesMap.put(Constants.WorkspaceProperties.METADATA_UUID, null); + + + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.SYNCH_FILTER, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.REMOTE_PATH, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.REMOTE_PERSISTENCE, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.TARGET_TOKEN, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.TBS, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.RELATED_CATALOG, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.VALIDATE_METADATA, null); + cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.ROOT_FOLDER_ID, null); + + + defaultConfigurationMap.put(Configuration.SCANNER_POOL_CORE_SIZE, "1"); + defaultConfigurationMap.put(Configuration.SCANNER_POOL_MAX_SIZE, "10"); + defaultConfigurationMap.put(Configuration.SCANNER_POOL_IDLE_MS, "30000"); + + defaultConfigurationMap.put(Configuration.TRANSFERS_POOL_CORE_SIZE, "1"); + defaultConfigurationMap.put(Configuration.TRANSFERS_POOL_MAX_SIZE, "10"); + defaultConfigurationMap.put(Configuration.TRANSFERS_POOL_IDLE_MS, "30000"); + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/FolderConfiguration.java b/src/main/java/org/gcube/usecases/ws/thredds/FolderConfiguration.java new file mode 100644 index 0000000..e15ac44 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/FolderConfiguration.java @@ -0,0 +1,40 @@ +package org.gcube.usecases.ws.thredds; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@RequiredArgsConstructor +@ToString +public class FolderConfiguration { + + public FolderConfiguration(FolderConfiguration other) { + this.providedMetadata=other.providedMetadata; + this.includeSubfolders=other.includeSubfolders; + this.publishingUserToken=other.publishingUserToken; + this.folderId=other.folderId; + this.catalogName=other.catalogName; + this.metadataFolderId=other.metadataFolderId; + } + + + + private boolean providedMetadata=false; + private boolean includeSubfolders=true; + + + @NonNull + private String publishingUserToken; + + @NonNull + private String folderId; + @NonNull + private String catalogName; + + + private String metadataFolderId=null; +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/FolderReport.java b/src/main/java/org/gcube/usecases/ws/thredds/FolderReport.java new file mode 100644 index 0000000..8c21115 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/FolderReport.java @@ -0,0 +1,31 @@ +package org.gcube.usecases.ws.thredds; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@ToString +@RequiredArgsConstructor +public class FolderReport { + + @NonNull + private FolderConfiguration config; + private Set transferredFiles=new HashSet<>(); + + /* map file->uuid*/ + private Map generatedMetadata=new HashMap<>(); + private Set publishedMetadata=new HashSet<>(); + + + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java b/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java new file mode 100644 index 0000000..10a2378 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/LocalConfiguration.java @@ -0,0 +1,44 @@ +package org.gcube.usecases.ws.thredds; + +import java.util.Properties; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; +@Slf4j +public class LocalConfiguration { + + private static LocalConfiguration instance=null; + + @Synchronized + private static final LocalConfiguration get() { + if(instance==null) + instance=new LocalConfiguration(); + return instance; + } + + public static String getProperty(String property) { + try{ + return (String) get().props.getOrDefault(property, Constants.defaultConfigurationMap.get(property)); + }catch(Throwable t) { + log.warn("Unable to get configuration property "+property,t); + return Constants.defaultConfigurationMap.get(property); + } + } + + + //***************** INSTANCE + + Properties props; + + public LocalConfiguration() { + props=new Properties(); + try{ + props.load(this.getClass().getResourceAsStream("configuration.properties")); + }catch(Exception e) { + log.warn("********************** UNABLE TO LOAD PROPERTIES **********************",e); + log.debug("Reverting to defaults : "+Constants.defaultConfigurationMap); + } + } + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java b/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java new file mode 100644 index 0000000..9627bec --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/NetUtils.java @@ -0,0 +1,57 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; + +import org.apache.commons.io.IOUtils; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class NetUtils { + + public static final File toFile(InputStream is) { + FileOutputStream outputStream =null; + try { + File toReturn=File.createTempFile("tempFile", ".tmp"); + outputStream=new FileOutputStream(toReturn); + + int read = 0; + byte[] bytes = new byte[1024]; + + while ((read = is.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } + return toReturn; + }catch(Throwable t) { + throw new RuntimeException(t); + }finally { + if(outputStream!=null) { + IOUtils.closeQuietly(outputStream); + } + } + } + + + public static String resolveRedirects(String url) throws IOException{ + log.debug("Resolving redirect for url {} ",url); + URL urlObj=new URL(url); + HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + int status=connection.getResponseCode(); + if(status>=300&&status<400){ + String newUrl=connection.getHeaderField("Location"); + log.debug("Following redirect from {} to {} ",url,newUrl); + return resolveRedirects(newUrl); + }else return url; + } + + + public static File download(String url) throws MalformedURLException, IOException { + return toFile(new URL(resolveRedirects(url)).openStream()); + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java b/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java new file mode 100644 index 0000000..035d661 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/PublishFolders.java @@ -0,0 +1,283 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.File; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WrongItemTypeException; +import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; +import org.gcube.usecases.ws.thredds.engine.PublishRequest; +import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode; +import org.gcube.usecases.ws.thredds.engine.PublishRequest.PublishItem; +import org.gcube.usecases.ws.thredds.engine.TransferRequestServer; +import org.gcube.usecases.ws.thredds.engine.TransferRequestServer.Report; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class PublishFolders { + + private static String WS_USER="fabio.sinibaldi"; + + + public static void main(String[] args) { + TokenSetter.set("/d4science.research-infrastructures.eu"); + HashSet configs=new HashSet<>(); + + +// String folderId="be451663-4d4f-4e23-a2c8-060cf15d83a7"; // NETCDF DATASETS + // String metadataFolderID="2de04273-ca79-4478-a593-354c5a12f942"; //metadata files + String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; // GP DDOS TEST + +// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET + + String publishingUserToken="***REMOVED***"; //fabio @NextNext + + + FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE"); + // folderConfig.setProvidedMetadata(true); + // folderConfig.setMetadataFolderId(metadataFolderID); + + + + + configs.add(folderConfig); + + + + + + + + TransferRequestServer server=new TransferRequestServer(); + for(FolderConfiguration entry:configs){ + try{ + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(WS_USER).getWorkspace(); + // FolderReport report=new FolderReport(entry); + log.info("Managing {} ",entry); + WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(entry.getFolderId()); + handleFolder(ws,entry,server,folder); + + + + }catch(WorkspaceException e){ + System.err.println("WORKSPACE EXC "); + e.printStackTrace(System.err); + }catch(HomeNotFoundException e){ + System.err.println("WORKSPACE EXC "); + e.printStackTrace(System.err); + }catch(InternalErrorException e){ + System.err.println("WORKSPACE EXC "); + e.printStackTrace(System.err); + }catch(UserNotFoundException e){ + System.err.println("WORKSPACE EXC "); + e.printStackTrace(System.err); + }catch(Exception e){ + System.err.println("UNEXPECTED EXC"); + e.printStackTrace(System.err); + } + } + System.out.println("Waiting for service.. "); + server.waitCompletion(); + + Report report=server.getReport(); + + File reportFile =report.toFile(folderConfig); + System.out.println("Report at "+reportFile.getAbsolutePath()); + + + } + + /** + * For *.nc | *.ncml + * if relatedMetadataFolder contains .xml use meta + * else ask DT to generate it + * + * + * @param wsFolder + * @param config + * @param server + * @return + * @throws WorkspaceException + * @throws InternalErrorException + * @throws ItemNotFoundException + * @throws HomeNotFoundException + * @throws UserNotFoundException + */ + + public static final void handleFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ + + log.info("Handling folder {} ",folder.getPath()); + + cleanUpForFileRemoval(ws, folder, config.getPublishingUserToken(),config.getCatalogName()); + + log.info("Folder {} cleaned up. Going to publish phase..",folder.getPath()); + publishFolder(ws,config,server,folder); + } + + + public static final void publishFolder(Workspace ws, FolderConfiguration config,TransferRequestServer server, WorkspaceFolder folder) throws InternalErrorException, ItemNotFoundException{ + + + + + //Access folder + List folderItems=folder.getChildren(); + + if(config.isIncludeSubfolders()) { + log.info("Going through subfolders first....."); + for(WorkspaceItem item:folderItems) { + try { + if(item.isFolder()) { + FolderConfiguration subConfig=new FolderConfiguration(config); + subConfig.setCatalogName(config.getCatalogName()+"/"+item.getName()); + publishFolder(ws,subConfig,server,(WorkspaceFolder) item); + log.debug("Subfolder {} successfully analyzed ",item.getPath()); + } + }catch(Exception e) { + try{ + log.warn("Unabel to check item {} ",item.getPath(),e); + }catch(InternalErrorException e1) { + log.warn("Unabel to check item and to get Path {} ",item,e); + } + } + } + } + + + + log.debug("Checking for ncml files .... "); + for(WorkspaceItem item : folderItems) { + try { + if(!item.isFolder()) { + String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); + if(prefix.equals(".ncml")) { + PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NCML, config.getCatalogName(), config.getPublishingUserToken()); + if(config.isProvidedMetadata()) { + String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; + File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); + if (meta!=null) req.setMetadata(meta); + } + // TODO NB Check for queue + server.put(req); + } + } + }catch(Exception e) { + try{ + log.warn("Unabel to check item {} ",item.getPath(),e); + }catch(InternalErrorException e1) { + log.warn("Unabel to check item and to get Path {} ",item,e); + } + } + } + + log.debug("Checking nc files.. "); + + for(WorkspaceItem item:folder.getChildren()){ + try { + if(!item.isFolder()) { + String prefix=item.getName().substring(item.getName().lastIndexOf("."), item.getName().length()); + if(prefix.equals(".nc")){ + // NC + PublishRequest req=new PublishRequest(new PublishItem(item),Mode.NC, config.getCatalogName(), config.getPublishingUserToken()); + if(config.isProvidedMetadata()) { + String toLookForName=item.getName().substring(0, item.getName().lastIndexOf(prefix))+".xml"; + File meta=getMetadataForDataset(ws, toLookForName, config.getMetadataFolderId()); + if (meta!=null) req.setMetadata(meta); + } + server.put(req); + } + } + }catch(Exception e) { + try{ + log.warn("Unabel to check item {} ",item.getPath(),e); + }catch(InternalErrorException e1) { + log.warn("Unabel to check item and to get Path {} ",item,e); + } + } + } + folder.getProperties().addProperties(Collections.singletonMap(Constants.LAST_UPDATE_TIME, System.currentTimeMillis()+"")); + log.debug("Creating requests... "); + } + + private static final File getMetadataForDataset(Workspace userWorkspace, String toLookForName, String metadataFolderId) throws WrongItemTypeException, InternalErrorException { + try{ + WorkspaceItem found=userWorkspace.find(toLookForName,metadataFolderId); + if(found==null) throw new ItemNotFoundException("Found item was null"); + return NetUtils.toFile(((ExternalFile)found).getData()); + }catch(ItemNotFoundException e) { + return null; + } + } + + // Remotely deletes folders which has history operations : RENAMING, REMOVAL + private static final void cleanUpForFileRemoval(Workspace ws, WorkspaceFolder folder,String targetToken, String remoteFolderPath) throws InternalErrorException { + List history=folder.getAccounting(); + long lastUpdateTimeMillis=getLastUpdateTime(folder); + Date lastUpdate=new Date(lastUpdateTimeMillis); + log.info("Checking history for {} (last update time {}) ",folder.getPath(),Commons.DATE_FORMAT.format(lastUpdate)); + + + //look into history + boolean toDeleteCurrentFolder=false; + for(AccountingEntry entry: history) { + Date eventTime=entry.getDate().getTime(); + + switch(entry.getEntryType()) { + case REMOVAL: + + case RENAMING:{ + log.debug("Found Accounting Entry [type : {}, date {}] ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); + if(eventTime.after(lastUpdate)) { + log.info("Found Accounting Entry [type : {}, date {}]. Removing remote folder. ",entry.getEntryType(),Commons.DATE_FORMAT.format(eventTime)); + toDeleteCurrentFolder=true; + } + } + } + if(toDeleteCurrentFolder) break; + } + + + //Delete Folder or scan children + if(toDeleteCurrentFolder) { + log.info("Deleting current folder {} from remote location {} ",folder.getPath(),remoteFolderPath); + try{ + Commons.cleanupFolder(remoteFolderPath,targetToken); + }catch(Throwable t) { + log.warn("Unable To cleanup folder {} . Remote Folder might not exists. If this is first publishing ignor this.",remoteFolderPath,t); + if(lastUpdateTimeMillis!=0) // do no rethrow in case of first publish + throw t; + + } + }else { + log.info("Folder is not to be cleaned up. Checking children.."); + for(WorkspaceItem item:folder.getChildren()) + if(item.isFolder())cleanUpForFileRemoval(ws, (WorkspaceFolder) item,targetToken, remoteFolderPath+"/"+item.getName()); + } + } + + private static long getLastUpdateTime(WorkspaceItem item) throws InternalErrorException { + try{ + return Long.parseLong(item.getProperties().getPropertyValue(Constants.LAST_UPDATE_TIME)); + }catch(Throwable e) { + log.debug("Unable to get last update time for {}. Considering 0..",item.getPath()); + return 0l; + } + } + + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java b/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java new file mode 100644 index 0000000..b8089d8 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/SyncEngine.java @@ -0,0 +1,57 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.File; +import java.util.Set; + +import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus; +import org.gcube.usecases.ws.thredds.engine.impl.SynchEngineImpl; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor; +import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; +import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo; +import org.gcube.usecases.ws.thredds.model.gui.CatalogBean; + +public interface SyncEngine { + + public static SyncEngine get() { + return SynchEngineImpl.get(); + } + + + public SyncFolderDescriptor check(String folderId, boolean recursively) throws WorkspaceInteractionException, InternalException; + public void registerCallBack(String folderId,SyncOperationCallBack callback) throws ProcessNotFoundException; + public ProcessDescriptor doSync(String folderId) throws WorkspaceInteractionException, InternalException; + public void stopSynch(String folderId) throws ProcessNotFoundException; + + public void setSynchronizedFolder(SynchFolderConfiguration config,String folderId) throws WorkspaceInteractionException, InternalException; + public void unsetSynchronizedFolder(String folderId,boolean deleteRemoteContent) throws WorkspaceInteractionException, InternalException; + + public SynchronizedElementInfo getInfo(String elementId); + + public void updateCatalogFile(String folderId, File toUpdate) throws InternalException; + + public void forceUnlock(String folderId)throws InternalException, WorkspaceInteractionException; + + + public void shutDown(); + + + public ProcessDescriptor getProcessDescriptorByFolderId(String folderId)throws ProcessNotFoundException; + public ProcessStatus getProcessStatusByFolderId(String folderId)throws ProcessNotFoundException; + + + + public void setRequestLogger(String path); + public boolean isRequestLoggerEnabled(); + public String getRequestLoggerPath(); + + + public Set getAvailableCatalogsByToken(String token) throws InternalException; + + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/TokenSetter.java b/src/main/java/org/gcube/usecases/ws/thredds/TokenSetter.java new file mode 100644 index 0000000..4253f33 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/TokenSetter.java @@ -0,0 +1,55 @@ +package org.gcube.usecases.ws.thredds; + +import static org.gcube.common.authorization.client.Constants.authorizationService; + +import java.util.Properties; + +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TokenSetter { + + private static Properties props=null; + + static{ + + } + + + public static synchronized void set(String scope){ + try{ + if(props==null) { + props=new Properties(); + try { + props.load(TokenSetter.class.getResourceAsStream("/tokens.properties")); + } catch (Exception e) { + throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION"); + } + } + if(!props.containsKey(scope)) throw new Exception("No token found for scope : "+scope); + SecurityTokenProvider.instance.set(props.getProperty(scope)); + }catch(Throwable e){ + log.trace("Unable to set token for scope "+scope,e); + } + ScopeProvider.instance.set(scope); + } + + + public static void setToken(String token){ + try{ + AuthorizationEntry entry = authorizationService().get(token); + ScopeProvider.instance.set(entry.getContext()); + SecurityTokenProvider.instance.set(token); + }catch(Throwable t) { + throw new RuntimeException("Unable to set token "+token,t); + } + } + + public static String getCurrentToken() { + return SecurityTokenProvider.instance.get(); + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishReport.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishReport.java new file mode 100644 index 0000000..120c804 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishReport.java @@ -0,0 +1,22 @@ +package org.gcube.usecases.ws.thredds.engine; + +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PublishReport { + + boolean isError=false; + + + private String sourceId; + private String sourceName; + + private TransferResult transferResult; + private MetadataReport metadataReport; + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java new file mode 100644 index 0000000..a740a09 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishRequest.java @@ -0,0 +1,64 @@ +package org.gcube.usecases.ws.thredds.engine; + +import java.io.File; +import java.util.HashSet; + +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Getter; +import lombok.NonNull; +import lombok.ToString; + +@Data +public class PublishRequest { + + @Getter + @AllArgsConstructor + @ToString + public static class PublishItem{ + String url; + String name; + String id; + + public PublishItem(WorkspaceItem item) throws InternalErrorException { + url=item.getPublicLink(false); + name=item.getName(); + id=item.getId(); + } + + } + + + public static enum Mode{ + NCML,NC + } + + @NonNull + private PublishItem source; + @NonNull + private Mode mode; + @NonNull + private String catalog; + @NonNull + private String publishToken; + + + private Integer queueCount=0; + private String queueId; + private File metadata=null; + + + private HashSet toGatherReportsId=null; + + public boolean isQueue() { + return queueCount>0; + } + + + public boolean isGenerateMeta() { + return metadata==null; + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java new file mode 100644 index 0000000..b9a53fe --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/PublishThread.java @@ -0,0 +1,217 @@ +package org.gcube.usecases.ws.thredds.engine; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.spatial.data.sdi.interfaces.Metadata; +import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions; +import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; +import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder; +import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; +import org.gcube.spatial.data.sdi.utils.ScopeUtils; +import org.gcube.usecases.ws.thredds.Commons; +import org.gcube.usecases.ws.thredds.NetUtils; +import org.gcube.usecases.ws.thredds.TokenSetter; +import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@RequiredArgsConstructor +@Slf4j +public class PublishThread implements Runnable { + + + + + + @NonNull + private PublishRequest request; + @NonNull + private ConcurrentHashMap reports; + + + private PublishReport publishReport; + // private Map reports; + // + + @Override + public void run() { + log.info("Request is {}",request); + log.debug("Switching from {} to {}",SecurityTokenProvider.instance.get(),request.getPublishToken()); + TokenSetter.setToken(request.getPublishToken()); + log.debug("Current scope is :{}, token is {} ",ScopeUtils.getCurrentScope(),SecurityTokenProvider.instance.get()); + + Destination dest=new Destination(); + dest.setPersistenceId("thredds"); + dest.setSubFolder("public/netcdf/"+request.getCatalog()); + dest.setOnExistingFileName(DestinationClashPolicy.REWRITE); + dest.setCreateSubfolders(true); + dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + String threddsHostName; + + + try { + threddsHostName = Commons.getThreddsHost(); + + DataTransferClient client=Commons.getDTClient(threddsHostName); + + + File toPublishSource=null; + + if(request.getMode().equals(Mode.NCML)) { + if(request.isQueue()){ + log.debug("Waiting for queue {}, expected Count {} ",request.getQueueId(),request.getQueueCount()); + waitFor(request.getQueueId(), request.getQueueCount()); + + log.debug("Loading netcdfFile .."); + File ncmlFile=NetUtils.download(request.getSource().getUrl()); + String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath())); + + for(String reportId:request.getToGatherReportsId()) { + PublishReport report=getReport(reportId); + //file://home/gcube/julien.barde/Workspace/DataMiner/Output_Data_Sets/Ichthyop2013.nc + + String toSetUrl="file:/"+report.getTransferResult().getRemotePath(); + toUpdateSource=toUpdateSource.replaceAll(reportId, toSetUrl); + } + + toPublishSource=File.createTempFile("nc_", ".ncml"); + PrintWriter out = new PrintWriter(toPublishSource); + out.write(toUpdateSource); + out.flush(); + + } + } + + + + TransferResult result; + + + // TODO NB Test run without metadata publication +// result=client.httpSource(request.getSource().getUrl(), dest); +// publishReport=new PublishReport(false,request.getSource().getId(),result,null); + + + if(!request.isGenerateMeta()) { + log.debug("Transfering before publishing meta.."); + + result = toPublishSource==null?client.httpSource(request.getSource().getUrl(), dest): + client.localFile(toPublishSource, dest); + + //NB DECOMMENT THIS!!!!!! +// Metadata meta=SDIAbstractPlugin.metadata().build(); +// +// log.debug("Publishing metadata.. "); +// +// MetadataPublishOptions opts=new MetadataPublishOptions(new TemplateInvocationBuilder().threddsOnlineResources(threddsHostName, request.getSource().getName(), request.getCatalog()).get()); +// opts.setGeonetworkCategory("Datasets"); +// MetadataReport report=meta.pushMetadata(request.getMetadata(), opts); + +// publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,report); + publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,new MetadataReport()); + + + }else { + log.debug("Metadata not provided.. "); + if(request.isQueue()&&request.getMode().equals(Mode.NC)) { + log.debug("Dataset file is linked in ncml, skipping metadata generation"); + result=client.httpSource(request.getSource().getUrl(), dest); + }else + result=client.httpSource(request.getSource().getUrl(), dest,new PluginInvocation("SIS/GEOTK")); + + + publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,null); + } + + + + + } catch (UnreachableNodeException | ServiceNotFoundException e) { + log.error("Unable to find Thredds. Publish scope is {} ",ScopeUtils.getCurrentScope(),e); + } catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException + | InvalidDestinationException | DestinationNotSetException e) { + log.error("Unable to transfer file, ",e); + } catch (IOException e) { + log.error("Unable to read/ write file. ",e); + } + + onCompletion(); + } + + + private void onCompletion() { + if(publishReport==null) publishReport=new PublishReport(true, request.getSource().getId(),request.getSource().getName(), null, null); + publishReport(publishReport); + + if(request.getMode().equals(Mode.NC)&&(request.isQueue())) { + alert(request.getQueueId(),request.getQueueCount()); + } + + } + + + private PublishReport getReport(String reportId) { + return reports.get(reportId); + } + + + private void publishReport(PublishReport report) { + reports.put(report.getSourceId(), report); + } + + + + + // static + + + private static ConcurrentHashMap semaphores=new ConcurrentHashMap<>(); + + + + private static void waitFor(String queueId,Integer expected) { + try { + log.debug("Waiting for queue {}. Expected Count is {} ",queueId,expected); + semaphores.getOrDefault(queueId, new Semaphore(expected*-1)).acquire(); + } catch (InterruptedException e) { + log.debug("Queue {} is completed."); + } + } + + private static void alert(String queueId, Integer expected) { + log.debug("Alerting queue {}. Expected count is {} ",queueId,expected); + Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1)); + sem.release(); + log.debug("Queue {} alerted. Remaining : {} out of {} ",queueId,sem.availablePermits(),expected); + } + + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/TransferRequestServer.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/TransferRequestServer.java new file mode 100644 index 0000000..a8a7c0e --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/TransferRequestServer.java @@ -0,0 +1,102 @@ +package org.gcube.usecases.ws.thredds.engine; + +import java.io.File; +import java.io.PrintWriter; +import java.util.Map.Entry; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.io.IOUtils; +import org.gcube.usecases.ws.thredds.FolderConfiguration; + +import lombok.Data; +import lombok.Synchronized; + +public class TransferRequestServer { + + @Data + public static class Report{ + private AtomicLong requestCount=new AtomicLong(0l); + private AtomicLong requestServed=new AtomicLong(0l); + private ConcurrentHashMap reports=new ConcurrentHashMap<>(); + + + public File toFile(FolderConfiguration configuration) { + return toFile(configuration,this); + } + + + private static final File toFile(FolderConfiguration config,Report report) { + PrintWriter writer =null; + try { + File toReturn=File.createTempFile("tempFile", ".tmp"); + writer=new PrintWriter(toReturn); + + writer.println("REPORT FOR WS-SYNCH"); + writer.println("Configuratiion was : "+config); + writer.println("Submitted runs : "+report.getRequestCount()); + writer.println("Item reports : "); + for(Entry entry: report.getReports().entrySet()) { + PublishReport rep=entry.getValue(); + writer.println("*********************************************************"); + if(rep.isError()) writer.println("OPERATION IS FAILED"); + writer.println("ITEM ID : "+rep.getSourceId()); + writer.println("ITEM NAME : "+rep.getSourceName()); + if(rep.getTransferResult()!=null)writer.println("Transfer report : "+rep.getTransferResult()); + if(rep.getMetadataReport()!=null)writer.println("Metadata report : "+rep.getMetadataReport()); + } + return toReturn; + }catch(Throwable t) { + throw new RuntimeException(t); + }finally { + if(writer!=null) { + IOUtils.closeQuietly(writer); + } + } + } + } + + private Report report=new Report(); + private ExecutorService service=null; + + public TransferRequestServer() { + BlockingQueue linkedBlockingDeque = new LinkedBlockingDeque( + 100); + service= new ThreadPoolExecutor(1, 10, 30, + TimeUnit.SECONDS, linkedBlockingDeque, + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + + public void put(PublishRequest request){ + System.out.println("Submitting transfer "+getReport().requestCount.incrementAndGet()); + service.execute(new PublishThread(request, getReport().getReports())); + +// service.execute(new RequestThread(baseUrl,filename,this,publishScope,toPublishMeta)); + } + @Synchronized + public Report getReport(){ + return report; + } + + + public void waitCompletion() { + boolean running=true; + service.shutdown(); + while(running){ + System.out.println("******************* WAITING FOR TERMINATION ***************** "); + try{ + running=!service.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES); + }catch(InterruptedException e){ + running=!service.isTerminated(); + } + } + System.out.println("Service is completed : "+service.isTerminated()); + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java new file mode 100644 index 0000000..81d1cfb --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/Process.java @@ -0,0 +1,312 @@ +package org.gcube.usecases.ws.thredds.engine.impl; + +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; + +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.data.transfer.model.RemoteFileDescriptor; +import org.gcube.usecases.ws.thredds.Constants; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status; +import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationRequest; +import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread; +import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest; +import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferToThreddsRequest; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; +import org.gcube.usecases.ws.thredds.model.CompletionCallback; +import org.gcube.usecases.ws.thredds.model.StepReport; +import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; +import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Process { + + + private boolean submittedRequests=false; + + private ProcessDescriptor descriptor; + private ProcessStatus status; + + private String processId=UUID.randomUUID().toString(); + + private Queue queuedReports=new LinkedList<>(); + + +// private String folderId; + + private WorkspaceFolderManager manager; + + private Set toInvokeCallbacks=ConcurrentHashMap.newKeySet(); + + private CompletionCallback callback=null; + + public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException { + log.debug("Created Process with id {} ",processId); +// this.folderId=folderId; + manager=new WorkspaceFolderManager(folderId); + manager.lock(processId); + SynchFolderConfiguration folderConfig=manager.getSynchConfiguration(); + + try { + descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().getPath(),System.currentTimeMillis(),processId,folderConfig); + }catch(Exception e) { + throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e); + } + + this.callback=callback; + + status=new ProcessStatus(); + } + + + public void launch(ExecutorService service) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalErrorException, InternalException { + WorkspaceUtils.resetStatus(manager.getTheFolder()); + status.setCurrentMessage("Analyzing folder.."); + + generateRequests(this,service, manager.getTheFolder()); + submittedRequests=true; + if(status.getQueuedTransfers().get()>0) { + status.setCurrentMessage("Waiting for requests ["+status.getQueuedTransfers().get()+"] to be served."); + status.setStatus(Status.ONGOING); + while(!queuedReports.isEmpty()) { + onStep(queuedReports.remove()); + } + }else { + status.setCurrentMessage("Folder is up to date."); + status.setStatus(Status.COMPLETED); + callback.onProcessCompleted(this); + invokeCallbacks(); + } + } + + + + public void addCallBack(SyncOperationCallBack toAddCallback) { + toInvokeCallbacks.add(toAddCallback); + log.debug("Added callback for process {}. Current callback size is {}",processId,toInvokeCallbacks.size()); + } + + public ProcessDescriptor getDescriptor() { + return descriptor; + } + + // signals from serving threads + + public void onStep(StepReport report) { + if(!submittedRequests) { + queuedReports.add(report); + }else { + // serve + updateStatus(report); + if(isCompleted()) { + try { + manager.setLastUpdateTime(); + }catch(Throwable t) { + log.error("Unable to update last update time.",t); + } + if(status.getStatus().equals(Status.WARNINGS)) + status.setCurrentMessage("Process completed with errors. Please check logs or retry."); + else status.setCurrentMessage("Synchronization complete."); + status.setStatus(Status.COMPLETED); + callback.onProcessCompleted(this); + } + invokeCallbacks(); + } + } + + private void invokeCallbacks() { + + for(SyncOperationCallBack callback:toInvokeCallbacks) { + try { + callback.onStep((ProcessStatus)status.clone(), (ProcessDescriptor)descriptor.clone()); + }catch(Throwable t) { + log.warn("Unable to invoke callback {}.",callback,t); + } + } + } + + + + + private boolean isCompleted() { + return (status.getErrorCount().get()+status.getServedTransfers().get()>=status.getQueuedTransfers().get()); + } + + @Synchronized + public void updateStatus(StepReport report) { + log.debug("Logging report {} ",report); + switch(report.getStatus()) { + case CANCELLED : + case ERROR:{ + status.getErrorCount().incrementAndGet(); + if(!status.getStatus().equals(Status.STOPPED)) + status.setStatus(Status.WARNINGS); + break; + } + default : { + status.getServedTransfers().incrementAndGet(); + break; + } + } + status.getLogBuilder().append( + String.format("%s - item [%s] %s: %s \n", Constants.DATE_FORMAT.format(new Date(report.getCompletionTime())), + report.getElementName(),report.getStatus()+"",report.getMessage())); + } + + + + public ProcessStatus getStatus() { + return status; + } + + + public void cancel() { + if(status.getQueuedTransfers().get()>1) { + status.setStatus(Status.STOPPED); + status.setCurrentMessage("Process Stopped. Waiting for remaining requests to cancel.."); + }else { + status.setStatus(Status.COMPLETED); + status.setCurrentMessage("Process cancelled before it started."); + } + invokeCallbacks(); + callback.onProcessCompleted(this); + + } + + public void cleanup() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + manager.unlock(processId); + } + + @Override + protected void finalize() throws Throwable { + try { + cleanup(); + }catch(Throwable t) { + log.warn("Exception while trying to cleanup {} ",this); + } + } + + + private static final void generateRequests(Process ownerProcess,ExecutorService service,WorkspaceFolder toScanFolder ) throws InternalErrorException, InternalException{ + + String folderPath=toScanFolder.getPath(); + + log.info("Generating requests for folder {}",folderPath); + log.debug("Process is {} ",ownerProcess.getDescriptor()); + Set handledWorkspaceItemEntries=new HashSet(); + + SynchFolderConfiguration config=ownerProcess.getDescriptor().getSynchConfiguration(); + + Set remoteChildrenNames; + Set localChildrenNames=new HashSet<>(); + + List localFolderChildren=toScanFolder.getChildren(); + for(WorkspaceItem item:localFolderChildren) { + localChildrenNames.add(item.getName()); + } + + String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken()); + + RemoteFileDescriptor folderDesc=null; + try{ + folderDesc=folderController.getFileDescriptor(); + }catch(RemoteFileNotFoundException e) { + log.debug("RemoteFolder {} doesn't exists. Creating it.. ",relativePath); + folderController.createEmptyFolder(null); + folderDesc=folderController.getFileDescriptor(); + } + + remoteChildrenNames=new HashSet<>(folderDesc.getChildren()); + + + //*********************** HANDLING ACCOUNTING ENTRIES + + Set handledAccountingEntries=WorkspaceUtils.scanAccountingForStatus( toScanFolder, config, localChildrenNames, remoteChildrenNames, folderController, ownerProcess, service); + + + + //SCAN FOLDER CONTENT + log.debug("Checking content of {} ",folderPath); + for(WorkspaceItem item:localFolderChildren) { + + if(item.isFolder()) { + // RECURSIVE ON SUB FOLDERS + generateRequests(ownerProcess,service,(WorkspaceFolder) item); + + }else { + Map props=item.getProperties().getProperties(); + String itemId=item.getId(); + String itemName=item.getName(); + + // REQUESTS ARE EVALUATED ON PROPERTIES (SET BY PREVIOUS SCAN) + + if(props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null)) { + try { + SynchronizationStatus status=SynchronizationStatus.valueOf(props.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)); + log.trace("Found TBS item {}, name {}, status : ",item.getId(),item.getName(),status); + SynchronizationRequest request=null; + + switch(status) { + case OUTDATED_REMOTE : request= new TransferToThreddsRequest(ownerProcess,toScanFolder,item); + break; + case OUTDATED_WS : request = new TransferFromThreddsRequest(ownerProcess, item, toScanFolder, null); + break; + + } + if(request!=null) { + // KEEP TRACK OF HANDLED ITEMS & LAUNCH + service.execute(new SynchronizationThread(request)); + log.debug("Submitted request number {} ",ownerProcess.status.getQueuedTransfers().incrementAndGet()); + }else log.debug("Item is up to date"); + handledWorkspaceItemEntries.add(itemName); + }catch(Throwable t) { + log.error("Unable to submit request for {} ID {} ",itemName,itemId,t); + } + } + } + } + + // check items to be imported + + try { + Set toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, handledAccountingEntries, handledWorkspaceItemEntries, toScanFolder, folderController, config, ownerProcess, service); + log.debug("Checking if remote location contains folders to be imported..."); + for(String item:toImportItems) { + if(folderController.getFileDescriptor(item).isDirectory()) { + log.info("Creating folder {} under {} ",item,folderPath); + try{ + WorkspaceFolder folder=toScanFolder.createFolder(item, "Imported from thredds"); + WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId()); + generateRequests(ownerProcess, service, folder); + }catch(Throwable t) { + log.error("Unable to import folder {} into {} ",item,folderPath); + } + } + } + }catch(InternalException e) { + log.error("Unable to check remote content with config {} ",config,e); + } + + + log.info("All requests for {} synchronization have been submitted [count {} ]. ",folderPath,ownerProcess.status.getQueuedTransfers().get()); + } + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessDescriptor.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessDescriptor.java new file mode 100644 index 0000000..e7bc75a --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessDescriptor.java @@ -0,0 +1,23 @@ +package org.gcube.usecases.ws.thredds.engine.impl; + +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class ProcessDescriptor implements Cloneable{ + + private String folderId; + private String folderPath; + private long launchTime; + private String processId; + + private SynchFolderConfiguration synchConfiguration; + + @Override + protected Object clone() throws CloneNotSupportedException { + return super.clone(); + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessStatus.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessStatus.java new file mode 100644 index 0000000..73309d6 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ProcessStatus.java @@ -0,0 +1,50 @@ +package org.gcube.usecases.ws.thredds.engine.impl; + +import java.util.concurrent.atomic.AtomicLong; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter + + +public class ProcessStatus implements Cloneable{ + + public static enum Status{ + INITIALIZING, // initial status + ONGOING, // synch in progress + WARNINGS, // errors occurred, still WORKING + STOPPED, // STOP received, waiting for request to finish + COMPLETED // FINISHED PROCESS + } + + + private AtomicLong queuedTransfers=new AtomicLong(0); + private AtomicLong servedTransfers=new AtomicLong(0); + private AtomicLong errorCount=new AtomicLong(0); + private Status status=Status.INITIALIZING; + + private StringBuilder logBuilder=new StringBuilder(); + + private String currentMessage="Waiting to start.."; + + @Override + protected Object clone() throws CloneNotSupportedException { + return super.clone(); + } + + @Override + public String toString() { + return "ProcessStatus [queuedTransfers=" + queuedTransfers + ", servedTransfers=" + servedTransfers + + ", errorCount=" + errorCount + ", status=" + status + ", currentMessage=" + currentMessage + "]"; + } + + public float getPercent() { + switch(status) { + case INITIALIZING : return 0; + case COMPLETED : return 1; + default : return queuedTransfers.get()==0?0:((float)(servedTransfers.get()+errorCount.get()))/queuedTransfers.get(); + } + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java new file mode 100644 index 0000000..8095f97 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/SynchEngineImpl.java @@ -0,0 +1,267 @@ +package org.gcube.usecases.ws.thredds.engine.impl; + +import java.io.File; +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.gcube.data.transfer.model.plugins.thredds.DataSetScan; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.usecases.ws.thredds.Constants; +import org.gcube.usecases.ws.thredds.LocalConfiguration; +import org.gcube.usecases.ws.thredds.SyncEngine; +import org.gcube.usecases.ws.thredds.engine.impl.threads.ProcessInitializationThread; +import org.gcube.usecases.ws.thredds.engine.impl.threads.RequestLogger; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceFolderNotRootException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; +import org.gcube.usecases.ws.thredds.model.CompletionCallback; +import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor; +import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; +import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo; +import org.gcube.usecases.ws.thredds.model.gui.CatalogBean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SynchEngineImpl implements SyncEngine{ + + private static SynchEngineImpl instance=null; + + + public static synchronized SyncEngine get() { + if(instance==null) { + instance=new SynchEngineImpl(); + } + return instance; + } + + + private SynchEngineImpl() { + + localProcesses=new ConcurrentHashMap<>(); + + int scannerMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_MAX_SIZE)); + int scannerCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_CORE_SIZE)); + int scannerIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.SCANNER_POOL_IDLE_MS)); + + + int transfersMaxSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_MAX_SIZE)); + int transfersCoreSize=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_CORE_SIZE)); + int transfersIdleMs=Integer.parseInt(LocalConfiguration.getProperty(Constants.Configuration.TRANSFERS_POOL_IDLE_MS)); + + initializationExecutor= new ThreadPoolExecutor(scannerCoreSize, scannerMaxSize, scannerIdleMs, + TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + + synchronizationExecutor=new ThreadPoolExecutor(transfersCoreSize, transfersMaxSize, transfersIdleMs, + TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + } + + + private String requestLoggerPath=null; + + @Override + public void setRequestLogger(String path) { + requestLoggerPath=path; + } + @Override + public boolean isRequestLoggerEnabled() { + return requestLoggerPath!=null; + } + @Override + public String getRequestLoggerPath() { + return requestLoggerPath; + } + + + //folder ID -> Process + private ConcurrentHashMap localProcesses; + + + private ExecutorService initializationExecutor=null; + + private ExecutorService synchronizationExecutor=null; + + + private final CompletionCallback completionCallback=new CompletionCallback() { + + @Override + public void onProcessCompleted(Process completedProcess) { + try { + ProcessDescriptor descriptor=completedProcess.getDescriptor(); + log.info("Process {} is completed. Going to cleanup.. ",descriptor); + localProcesses.remove(descriptor.getFolderId()); + completedProcess.cleanup(); + }catch(Throwable t) { + log.warn("Unable to cleanup {} ",completedProcess,t); + } + } + }; + + + + @Override + public SyncFolderDescriptor check(String folderId, boolean recursively) throws WorkspaceInteractionException, InternalException { + WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId); + return manager.check(recursively); + } + + @Override + public void registerCallBack(String folderId, SyncOperationCallBack callback) throws ProcessNotFoundException { + if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under local processes"); + localProcesses.get(folderId).addCallBack(callback); + } + + @Override + public ProcessDescriptor doSync(String folderId) throws WorkspaceInteractionException, InternalException { + if(localProcesses.containsKey(folderId)) + return localProcesses.get(folderId).getDescriptor(); + else { + WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId); + if (!manager.isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not configured for synchronization."); + if(manager.isLocked()) throw new WorkspaceLockedException("Folder "+folderId+"is locked by an external process."); + if(!manager.isRoot()) throw new WorkspaceFolderNotRootException("Unable to launch synch operation. Folder "+folderId+" is not root configuration"); + Process toLaunch=new Process(folderId,completionCallback); + localProcesses.put(folderId, toLaunch); + initializationExecutor.submit(new ProcessInitializationThread(toLaunch,synchronizationExecutor)); + return toLaunch.getDescriptor(); + } + } + + @Override + public void stopSynch(String folderId) throws ProcessNotFoundException { + if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under local processes"); + localProcesses.get(folderId).cancel(); + } + + + @Override + public void setSynchronizedFolder(SynchFolderConfiguration config,String folderId) throws WorkspaceInteractionException, InternalException { + + // Check config + if(config==null) throw new InternalException("Passed config is null : "+config); + String remotePath=config.getRemotePath(); + if(remotePath==null||remotePath.isEmpty()||remotePath.startsWith("/")) + throw new InternalException("Invalid remote path "+remotePath+"."); + + new WorkspaceFolderManager(folderId).configure(config); + } + + @Override + public void unsetSynchronizedFolder(String folderId,boolean deleteRemoteContent) throws WorkspaceInteractionException, InternalException { + new WorkspaceFolderManager(folderId).dismiss(deleteRemoteContent); + } + + @Override + public SynchronizedElementInfo getInfo(String elementId) { + return WorkspaceFolderManager.getInfo(elementId); + } + + @Override + public void updateCatalogFile(String folderId, File toUpdate) throws InternalException { + File previousCatalogFile=null; + try { + WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId); + previousCatalogFile=manager.loadCatalogFile(); + String lockId=UUID.randomUUID().toString(); + manager.lock(lockId); + manager.updateCatalogFile(toUpdate); + manager.unlock(lockId); + }catch(Throwable t) { + log.warn("Unable to update catalogFile for {}. Trying to restore previous one..",folderId,t); + throw new InternalException("Unable to restore previous catalog.",t); + //TODO try to restore previous catalog + } + } + + @Override + public void shutDown() { + log.trace("Cancelling processes..."); + for(Entry entry:localProcesses.entrySet()) + entry.getValue().cancel(); + + log.trace("Shutting down services... "); + initializationExecutor.shutdown(); + synchronizationExecutor.shutdown(); + + do { + log.trace("Waiting for services to terminate.."); + try {Thread.sleep(1000l); + } catch (InterruptedException e) {} + }while(!initializationExecutor.isTerminated()||!synchronizationExecutor.isTerminated()); + + RequestLogger.get().close(); + log.trace("Terminated."); + } + + @Override + public void forceUnlock(String folderId) throws InternalException, WorkspaceInteractionException { + log.warn("Forcing unlock of {} ",folderId); + new WorkspaceFolderManager(folderId).forceUnlock(); + } + + + @Override + public ProcessDescriptor getProcessDescriptorByFolderId(String folderId) throws ProcessNotFoundException { + if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under processes or process is not in this host"); + return localProcesses.get(folderId).getDescriptor(); + } + + @Override + public ProcessStatus getProcessStatusByFolderId(String folderId) throws ProcessNotFoundException { + if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under processes or process is not in this host"); + return localProcesses.get(folderId).getStatus(); + } + + + @Override + public Set getAvailableCatalogsByToken(String token) throws InternalException { + ThreddsController controller=new ThreddsController("",token); + ThreddsInfo info=controller.getThreddsInfo(); + Set toReturn=asCatalogBeanSet(info.getCatalog()); + DataSetScan mainScan=info.getCatalog().getDeclaredDataSetScan().iterator().next(); + CatalogBean defaultBean=new CatalogBean(mainScan.getName(),mainScan.getLocation(),true); + toReturn.remove(defaultBean); + toReturn.add(defaultBean); + + //*** Cleaning : + // absolute paths to relative paths (from thredds persistence) + // leading/ending '/' + String threddsPersistencePath=info.getLocalBasePath(); + for(CatalogBean bean:toReturn) { + String path=bean.getPath(); + if(path.startsWith(threddsPersistencePath)) + path=path.substring(threddsPersistencePath.length()); + if(path.startsWith("/")) path=path.substring(1); + if(path.endsWith("/"))path=path.substring(0, path.length()-1); + bean.setPath(path); + } + + + return toReturn; + } + + private static HashSet asCatalogBeanSet(ThreddsCatalog catalog){ + HashSet toReturn=new HashSet<>(); + for(DataSetScan scan:catalog.getDeclaredDataSetScan()) + toReturn.add(new CatalogBean(scan.getName(), + scan.getLocation(),false)); + if(catalog.getSubCatalogs()!=null&&catalog.getSubCatalogs().getLinkedCatalogs()!=null) + for(ThreddsCatalog sub:catalog.getSubCatalogs().getLinkedCatalogs()) + toReturn.addAll(asCatalogBeanSet(sub)); + return toReturn; + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java new file mode 100644 index 0000000..a218a0b --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/ThreddsController.java @@ -0,0 +1,318 @@ +package org.gcube.usecases.ws.thredds.engine.impl; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.util.Set; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.tika.io.IOUtils; +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.library.client.AuthorizationFilter; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.model.RemoteFileDescriptor; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.spatial.data.sdi.model.ServiceConstants; +import org.gcube.spatial.data.sdi.utils.ScopeUtils; +import org.gcube.usecases.ws.thredds.Constants; +import org.gcube.usecases.ws.thredds.TokenSetter; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; +import org.gcube.usecases.ws.thredds.faults.UnableToLockException; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ThreddsController { + + private String hostname; + private String operatingPath; + private String targetToken; + + public ThreddsController(String path,String targetToken) throws InternalException { + operatingPath=path; + this.targetToken=targetToken; + setTargetToken(); + hostname=getThreddsHost(); + if(hostname==null) throw new InternalException("Unable to find a thredds instance in target scope "+ScopeUtils.getCurrentScope()); + resetCallerToken(); + } + + private static final String truncate(String toTruncate) { + return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"..."; + } + + private String callerToken=null; + + private void setTargetToken() { + if(callerToken==null) { + callerToken=TokenSetter.getCurrentToken(); + log.trace("Storing caller token {}. Target Token is {}",truncate(callerToken),truncate(targetToken)); + TokenSetter.setToken(targetToken); + }else { + log.trace("Caller token {} already registered. Target Token is {}",truncate(callerToken),truncate(targetToken)); + } + } + + private void resetCallerToken() { + if(callerToken!=null) { + log.trace("Resetting caller token {}. Target Token is {}, current is {} ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())); + TokenSetter.setToken(callerToken); + callerToken=null; + }else log.trace("Caller token {} already reset [current token {}]. Target Token is {}",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)); + } + + public final ThreddsInfo getThreddsInfo() { + setTargetToken(); + try{ + String infoPath="https://"+hostname+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";; + log.info("Loading thredds info from {} ",infoPath); + WebTarget target=getWebClient().target(infoPath); + return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class); + }finally { + resetCallerToken(); + } + } + + + public void lockFolder(String processId) throws UnableToLockException { + setTargetToken(); + PrintWriter writer=null; + File temp=null; + try{ + log.info("Locking remote path {} to processId {} ",operatingPath,processId); + DataTransferClient cl=getDTClient(hostname); + + Destination dest=new Destination(); + dest.setCreateSubfolders(true); + dest.setOnExistingFileName(DestinationClashPolicy.FAIL); + dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + dest.setPersistenceId(Constants.THREDDS_PERSISTENCE); + dest.setSubFolder(operatingPath); + dest.setDestinationFileName(Constants.LOCK_FILE); + + temp=File.createTempFile("tmp_lock", ".tmp"); + writer=new PrintWriter(temp); + writer.write(processId); + writer.flush(); + writer.close(); + + cl.localFile(temp, dest); + }catch(Throwable t) { + throw new UnableToLockException("Unable to lock "+operatingPath,t); + }finally { + if(writer!=null) IOUtils.closeQuietly(writer); + if(temp!=null)try { + Files.deleteIfExists(temp.toPath()); + }catch(IOException e) { + log.warn("Unable to delete temp file {} ",temp.getAbsolutePath(),e); + } + resetCallerToken(); + } + + } + + public void deleteThreddsFile(String location) throws RemoteFileNotFoundException { + setTargetToken(); + String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location); + log.info("Deleting file at {} ",urlString); + try{ + getWebClient().target(urlString).request().delete(); + }catch(Throwable t) { + throw new RemoteFileNotFoundException("Unable to access "+urlString, t); + }finally{ + resetCallerToken(); + } + } + + + public boolean existsThreddsFile(String location) { + try{ + getFileDescriptor(location); + return true; + }catch(RemoteFileNotFoundException e) { + return false; + } + } + + public String readThreddsFile(String location) throws RemoteFileNotFoundException { + setTargetToken(); + String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location); + log.info("Reading file at {} ",urlString); + try{ + return getWebClient().target(urlString).request().get().readEntity(String.class); + }catch(Throwable t) { + throw new RemoteFileNotFoundException("Unable to access "+urlString, t); + }finally { + resetCallerToken(); + } + } + + + public void createEmptyFolder(String targetPath) throws InternalException { + setTargetToken(); + String toCleanPath=getPathFromStartingLocation(targetPath); + try{ + log.info("Cleaning up {} on {} ",toCleanPath,hostname); + DataTransferClient client=getDTClient(hostname); + File toTransfer=File.createTempFile("clean", ".dt_temp"); + toTransfer.createNewFile(); + Destination dest=new Destination(); + dest.setCreateSubfolders(true); + dest.setOnExistingFileName(DestinationClashPolicy.REWRITE); + dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE); + dest.setPersistenceId(Constants.THREDDS_PERSISTENCE); + dest.setSubFolder(toCleanPath); + dest.setDestinationFileName(toTransfer.getName()); + + log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname); + client.localFile(toTransfer, dest); + this.deleteThreddsFile(targetPath+"/"+toTransfer.getName()); + log.info("Done"); + log.debug("Resulting folder descriptor : {} ",getFileDescriptor(targetPath)); + }catch(Exception e) { + log.error("Unable to delete remote folder "+toCleanPath,e); + throw new InternalException("Unable to cleanup remote folder."); + }finally { + resetCallerToken(); + } + } + + public ThreddsCatalog createCatalog(String name) throws InternalException { + setTargetToken(); + try{ + log.info("Creating catalog with name {} for path {} ",name,operatingPath); + String sdiUrl="http://"+getSDIServiceHost()+"/"+Constants.SDI_THREDDS_BASE_URL; + Response resp=getWebClient().target(sdiUrl). + queryParam("name", name). + queryParam("path", operatingPath). + queryParam("folder", operatingPath).request().put(null); + if(!(resp.getStatus()>=200&&resp.getStatus()<300)) + throw new InternalException("Failed catalog registration on SDI Service. Message "+resp.readEntity(String.class)); + return getCatalog(); + }catch(Throwable t) { + log.error("Unable to create catalog",t); + throw new InternalException("Unable to create catalog",t); + }finally { + resetCallerToken(); + } + } + + public ThreddsCatalog getCatalog() { + setTargetToken(); + try{ThreddsInfo info=getThreddsInfo(); + String instanceBasePath=info.getLocalBasePath(); + return info.getCatalogByFittingLocation(instanceBasePath+"/"+operatingPath); + }finally { + resetCallerToken(); + } + } + + + public RemoteFileDescriptor getFileDescriptor() throws RemoteFileNotFoundException { + return getFileDescriptor(null); + } + + public RemoteFileDescriptor getFileDescriptor(String path) throws RemoteFileNotFoundException { + setTargetToken(); + String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path); + log.info("Reading file at {} ",urlString); + try{ + return getWebClient().target(urlString).queryParam("descriptor", true).request().get().readEntity(RemoteFileDescriptor.class); + }catch(Throwable t) { + throw new RemoteFileNotFoundException("Unable to access "+urlString, t); + }finally { + resetCallerToken(); + } + } + + public InputStream getInputStream(String path) throws RemoteFileNotFoundException { + setTargetToken(); + String urlString="http://"+hostname+":80/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path); + log.info("Reading file at {} ",urlString); + try{ + return getWebClient().target(urlString).request().get().readEntity(InputStream.class); + }catch(Throwable t) { + throw new RemoteFileNotFoundException("Unable to access "+urlString, t); + }finally { + resetCallerToken(); + } + } + + + public TransferResult transferFile(Destination dest,String url,Set invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { + setTargetToken(); + try{DataTransferClient client=getDTClient(hostname); + if(invocations!=null&&!invocations.isEmpty()) + return client.httpSource(url, dest,invocations); + else return client.httpSource(url, dest); + }finally { + resetCallerToken(); + } + } + + + private String getPathFromStartingLocation(String location) { + if(location!=null&&location.length()>0)return operatingPath+"/"+location; + else return operatingPath; + } + + private static String getSDIServiceHost(){ + return getGCoreEndpointHostname(ServiceConstants.SERVICE_CLASS, ServiceConstants.SERVICE_NAME); + } + + + private static String getThreddsHost(){ + return getGCoreEndpointHostname(ServiceConstants.SERVICE_CLASS, "Thredds"); + } + + private static String getGCoreEndpointHostname(String serviceClass,String serviceName) { + SimpleQuery query =queryFor(GCoreEndpoint.class); + query.addCondition("$resource/Profile/ServiceClass/text() eq '"+serviceClass+"'") + .addCondition("$resource/Profile/ServiceName/text() eq '"+serviceName+"'"); + // .setResult("$resource/Profile/AccessPoint"); + + DiscoveryClient client = clientFor(GCoreEndpoint.class); + + GCoreEndpoint endpoint= client.submit(query).get(0); + + return endpoint.profile().endpoints().iterator().next().uri().getHost(); + } + + private static Client getWebClient() { + return ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class)) + .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true); + } + + private static DataTransferClient getDTClient(String threddsHostName) throws UnreachableNodeException, ServiceNotFoundException { + log.debug("Getting DT Client for {} ",threddsHostName); + return DataTransferClient.getInstanceByEndpoint("http://"+threddsHostName+":80"); + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java new file mode 100644 index 0000000..e46cefc --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceFolderManager.java @@ -0,0 +1,351 @@ +package org.gcube.usecases.ws.thredds.engine.impl; + +import java.io.File; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceException; +import org.gcube.data.transfer.model.RemoteFileDescriptor; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsCatalog; +import org.gcube.usecases.ws.thredds.Constants; +import org.gcube.usecases.ws.thredds.engine.impl.threads.ProcessIdProvider; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.LockNotOwnedException; +import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; +import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; +import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo; +import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class WorkspaceFolderManager { + + + + public static SynchronizedElementInfo getInfo(String elementId) { + // TODO FILL THIS + return null; + } + + + private WorkspaceFolder theFolder; + + + private String folderId; + + // Cahced objects + private SynchFolderConfiguration config=null; + private ThreddsController threddsController=null; + + private Workspace ws; + + public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException { + try{ + ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + theFolder=(WorkspaceFolder) ws.getItem(folderId); + this.folderId=folderId; + }catch(WorkspaceException | InternalErrorException | HomeNotFoundException | UserNotFoundException e) { + throw new WorkspaceInteractionException("Unable to access folder id "+folderId,e); + } + } + + + public WorkspaceFolder getTheFolder() { + return theFolder; + } + + public ThreddsController getThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + if(threddsController==null) { + SynchFolderConfiguration config=getSynchConfiguration(); + threddsController=new ThreddsController(config.getRemotePath(),config.getTargetToken()); + } + return threddsController; + } + + private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + try { + WorkspaceFolder root=(WorkspaceFolder) ws.getItem(getSynchConfiguration().getRootFolderId()); + SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root); + return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken()); + }catch(WorkspaceException | InternalErrorException e) { + throw new WorkspaceInteractionException(e); + } + + } + + public boolean isRoot() throws WorkspaceNotSynchedException, WorkspaceInteractionException{ + try{ + return getSynchConfiguration().getRootFolderId().equals(theFolder.getId()); + }catch(InternalErrorException e) { + throw new WorkspaceInteractionException(e); + } + } + + + public SynchFolderConfiguration getSynchConfiguration() throws WorkspaceInteractionException, WorkspaceNotSynchedException { + if(config==null) { + try { + if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched."); + log.debug("Loading properties for "); + config=WorkspaceUtils.loadConfiguration(theFolder); + }catch(InternalErrorException e) { + throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e); + } + } + return config; + } + + public boolean isSynched() throws WorkspaceInteractionException { + try{ + Map props=theFolder.getProperties().getProperties(); + return props.containsKey(Constants.WorkspaceProperties.TBS)&&(props.get(Constants.WorkspaceProperties.TBS)!=null); + }catch(InternalErrorException e) { + throw new WorkspaceInteractionException("Unable to check Synch flag on "+folderId,e); + } + } + + + public SyncFolderDescriptor check(boolean recursively) throws WorkspaceInteractionException, InternalException { + if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched."); + if(isLocked()&&!isLockOwned())throw new WorkspaceLockedException("Workspace "+folderId+" is locked."); + + SynchFolderConfiguration config=getSynchConfiguration(); + try{ + checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder)); + return new SyncFolderDescriptor(this.folderId,this.theFolder.getPath(),config); + }catch(InternalErrorException e) { + throw new WorkspaceInteractionException(e); + } + } + + private boolean isLockOwned() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + String currentProcessId=ProcessIdProvider.instance.get(); + if(currentProcessId==null) return false; + return currentProcessId.equals(getLockId()); + } + + + public String getLockId() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + return getRootThreddsController().readThreddsFile(Constants.LOCK_FILE); + } + + public boolean isLocked() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + return getRootThreddsController().existsThreddsFile(Constants.LOCK_FILE); + } + + + public void configure(SynchFolderConfiguration toSet) throws WorkspaceInteractionException, InternalException { + if(isSynched()) throw new WorkspaceInteractionException("Folder "+folderId+" is already configured for synchronization."); + log.info("Configuring folder {} as {} ",folderId,toSet); + + + + // Checking AND initializing remote folder + log.debug("Checking remote folder existence .. "); + boolean createCatalog=false; + try { + String catalogName=toSet.getToCreateCatalogName(); + + ThreddsController controller= new ThreddsController(toSet.getRemotePath(),toSet.getTargetToken()); + if(!controller.existsThreddsFile(null)) { + log.info("Folder not found, creating it.."); + controller.createEmptyFolder(null); + createCatalog=true; + }else { + ThreddsCatalog catalog=controller.getCatalog(); + if (catalog==null) { + createCatalog=true; + }else { + log.info("Found matching catalog {} ",catalog); + catalogName=catalog.getTitle(); + if(catalogName==null) catalogName=catalog.getDeclaredDataSetScan().iterator().next().getName(); + toSet.setToCreateCatalogName(catalogName); + } + } + + if(createCatalog) { + log.info("Creating catalog {} ",catalogName); + log.debug("Created catalog {}", controller.createCatalog(catalogName)); + } + + WorkspaceUtils.initProperties(theFolder, toSet.getRemotePath(), toSet.getFilter(), + toSet.getTargetToken(),toSet.getToCreateCatalogName(),toSet.getValidateMetadata(),theFolder.getId()); + + }catch(InternalException e) { + throw new InternalException ("Unable to check/initialize remote folder",e); + }catch(InternalErrorException e) { + throw new WorkspaceInteractionException("Unable to set Properties to "+folderId,e); + } + } + + public void dismiss(boolean deleteRemote) throws WorkspaceInteractionException, InternalException { + if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched."); + if(isLocked()&&!isLockOwned())throw new WorkspaceLockedException("Workspace "+folderId+" is locked."); + + try { + cleanCache(); + WorkspaceUtils.cleanItem(theFolder); + if(deleteRemote) + getThreddsController().createEmptyFolder(null); + }catch(InternalErrorException e) { + throw new WorkspaceInteractionException("Unable to cleanup "+folderId,e); + } + } + + public void setLastUpdateTime() throws InternalErrorException { + WorkspaceUtils.setLastUpdateTime(theFolder, System.currentTimeMillis()); + } + + + public void forceUnlock() throws InternalException,WorkspaceInteractionException { + try { + getRootThreddsController().deleteThreddsFile(Constants.LOCK_FILE); + } catch (RemoteFileNotFoundException e) { + log.debug("Forced unlock but no file found.",e); + } catch (WorkspaceNotSynchedException e) { + log.warn("Invoked force lock on not synched folder.",e); + } catch (InternalException | WorkspaceInteractionException e) { + throw e ; + } + } + + + public void lock(String processId) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + getRootThreddsController().lockFolder(processId); + } + + public void unlock() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + unlock(ProcessIdProvider.instance.get()); + } + + public void unlock(String processId) throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException { + String currentLock=getLockId(); + if(processId.equals(currentLock)) getRootThreddsController().deleteThreddsFile(Constants.LOCK_FILE); + else throw new LockNotOwnedException("Process "+processId+" can't remove lock owned by "+currentLock); + } + + + //*************************** PRIVATE + + private void cleanCache() { + this.config=null; + this.threddsController=null; + } + + + + + + private static void checkFolder(WorkspaceFolder folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws InternalErrorException, InternalException { + // Check folder configuration + log.trace("Checking folder {} ",folder.getPath()); + log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder); + + String folderName=folder.getName(); + + String currentRemotePath=rootConfig.getRemotePath()+((relativePathFromRootFolder==null)?"":"/"+relativePathFromRootFolder); + + + + ThreddsController controller=new ThreddsController(currentRemotePath, rootConfig.getTargetToken()); + + + HashSet currentFolderExistingItem=new HashSet(); + + + log.debug("Initializing properties for {} ",folderName); + //INIT PROPERTIES IF NOT PRESENT + if(!WorkspaceUtils.isConfigured(folder)) + WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId); + + for(WorkspaceItem item:folder.getChildren()) { + String itemName=item.getName(); + String itemRelativePath=(relativePathFromRootFolder==null)?itemName:relativePathFromRootFolder+"/"+itemName; + String itemRemotePath=currentRemotePath+"/"+itemName; + if(item.isFolder()) { + if(recursive) + checkFolder((WorkspaceFolder) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine); + else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId); + }else if(rootConfig.matchesFilter(itemName)) { + if(!WorkspaceUtils.isConfigured(item)) + WorkspaceUtils.initProperties(item, null, null, null,null,null,null); + } + currentFolderExistingItem.add(itemName); + } + + + // ACTUALLY CHECK STATUS + + if(controller.existsThreddsFile(null)) { + SynchronizationStatus folderStatus=SynchronizationStatus.OUTDATED_REMOTE; + log.debug("Remote Folder {} exists. Checking status..",currentRemotePath); + RemoteFileDescriptor folderDesc=controller.getFileDescriptor(); + HashSet remoteFolderItems=new HashSet<>(folderDesc.getChildren()); + + + + + // CHECK HISTORY + Set accountingEntries=WorkspaceUtils.scanAccountingForStatus(folder, rootConfig, + currentFolderExistingItem, remoteFolderItems, + controller, null, null); + if(accountingEntries.isEmpty()) { + log.debug("No accounting entries found"); + folderStatus=SynchronizationStatus.UP_TO_DATE; + } + + // CHECK WS ITEMS + for(WorkspaceItem item:folder.getChildren()) + if(item.isFolder()||rootConfig.matchesFilter(item.getName())) { + SynchronizationStatus itemStatus=WorkspaceUtils.getStatusAgainstRemote(item, remoteFolderItems, controller,lastUpdatedRoutine); + item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,itemStatus+"")); + folderStatus=folderStatus.equals(SynchronizationStatus.UP_TO_DATE)?itemStatus:folderStatus; + + } + + // CHECK REMOTE FOLDER + if(folderStatus.equals(SynchronizationStatus.UP_TO_DATE)) { + Set toImportItems=WorkspaceUtils.scanRemoteFolder(folderDesc, accountingEntries, currentFolderExistingItem, folder, controller, rootConfig, null, null); + if(!toImportItems.isEmpty()) folderStatus=SynchronizationStatus.OUTDATED_WS; + } + + folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,folderStatus+"")); + }else { + // Remote Folder not existing, set everything to OUTDATED_REMOTE + for(WorkspaceItem item:folder.getChildren()) + item.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"")); + folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.OUTDATED_REMOTE+"")); + } + + + + } + + public File loadCatalogFile() { + // TODO Auto-generated method stub + return null; + } + + + public void updateCatalogFile(File toUpload) { + // TODO Auto-generated method stub + + } + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java new file mode 100644 index 0000000..85bd453 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/WorkspaceUtils.java @@ -0,0 +1,356 @@ +package org.gcube.usecases.ws.thredds.engine.impl; + +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; + +import org.gcube.common.homelibary.model.items.type.WorkspaceItemType; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.workspace.Properties; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; +import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRemoval; +import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntryRenaming; +import org.gcube.data.transfer.model.RemoteFileDescriptor; +import org.gcube.usecases.ws.thredds.Constants; +import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest; +import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread; +import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; +import org.gcube.usecases.ws.thredds.model.StepReport; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; +import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class WorkspaceUtils { + + + + + + /** + * Checks current accounting info in order to infere synchronization status. + * OwnerProcess and service parameters can be null for check purposes. + * + * @param folderPath + * @param toScanFolder + * @param config + * @param localChildrenNames + * @param remoteChildrenNames + * @param folderController + * + * @param ownerProcess + * @param service + * @return set of Item names that have been found to be synchronized + * @throws InternalErrorException + */ + static Set scanAccountingForStatus( + WorkspaceFolder toScanFolder, + SynchFolderConfiguration config, + Set localChildrenNames, + Set remoteChildrenNames, + ThreddsController folderController, + + Process ownerProcess, + ExecutorService service) throws InternalErrorException{ + + Set handledAccountingEntries=new HashSet<>(); + + log.debug("Checking history of {} ",toScanFolder.getPath()); + + + String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + + + Date folderLastUpdateTime=null; + try{ + folderLastUpdateTime=WorkspaceUtils.safelyGetLastUpdate(toScanFolder); + }catch(Throwable t) { + log.warn("Unable to get folder {} last update time. Assuming first run.. ",toScanFolder.getName(),t); + folderLastUpdateTime=new Date(0l); + } + // scanning for deletions + log.debug("Checking Accounting for {}. Last update time is {} ",toScanFolder.getName(),Constants.DATE_FORMAT.format(folderLastUpdateTime)); + for(AccountingEntry entry:toScanFolder.getAccounting()) { + try { + Date eventTime=entry.getDate().getTime(); + if(folderLastUpdateTime==null|| eventTime.after(folderLastUpdateTime)) { // SKIP IF ENTRY OLDER THAN LAST UPDATE TIME + String toDeleteRemote=null; + switch(entry.getEntryType()) { + case CUT: + case REMOVAL:{ + AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry; + if(removalEntry.getItemType().equals(WorkspaceItemType.FOLDER)|| + config.matchesFilter(removalEntry.getItemName())) + toDeleteRemote=removalEntry.getItemName(); + break; + } + case RENAMING:{ + AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry; + WorkspaceItem newItem=toScanFolder.find(renamingEntry.getNewItemName()); + if(newItem.isFolder()||config.matchesFilter(renamingEntry.getOldItemName())) + toDeleteRemote=renamingEntry.getOldItemName(); + break; + } + } + + + if(toDeleteRemote!=null){ + // SKIP IF LOCAL EXISTS + if(localChildrenNames.contains(toDeleteRemote)) + log.debug("Skipping accounting entry for existing local item {} ",toDeleteRemote); + else if(remoteChildrenNames.contains(toDeleteRemote)) { + log.debug("Checking age of remote {} ",toDeleteRemote); + + // IF REMOTE OLDER THAN ENTRY -> DELETE REQUEST + // IF REMOTE NEWER -> IMPORT REQUEST + RemoteFileDescriptor remote=folderController.getFileDescriptor(relativePath+"/"+toDeleteRemote); + Date remoteDate=new Date(remote.getLastUpdate()); + log.debug("Last remote update : {} . Event date {} ",Constants.DATE_FORMAT.format(remoteDate),Constants.DATE_FORMAT.format(eventTime)); + if(service!=null) { + log.debug("Service is not null. Submitting request ... "); + if(eventTime.after(remoteDate)) { + service.execute(new SynchronizationThread(new DeleteRemoteRequest(ownerProcess, toScanFolder,toDeleteRemote))); + handledAccountingEntries.add(toDeleteRemote); + log.debug("Submitted DELETION request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet()); + } +// }else { +// service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, toDeleteRemote))); +// log.debug("Submitted UPDATE-LOCAL request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet()); +// } + } + }else log.debug("To delete remote {} not found. skipping it.. ",toDeleteRemote); + // SKIP IF REMOTE NOT FOUND + } + } + }catch(Throwable t) { + log.error("Unable to submit deletion request for {} ",entry,t); + } + } + + return handledAccountingEntries; + } + + + /** + * Scans remote Folder in order to gather elements to be synchronized. + * OwnerProcess and Service can be null for check purposes. + * + * + * @param folderPath + * @param folderDesc + * @param handledAccountingEntries + * @param handledWorkspaceItemEntries + * @param toScanFolder + * @param folderController + * @param config + * @param ownerProcess + * @param service + * @return + * @throws InternalException + * @throws InternalErrorException + */ + static Set scanRemoteFolder( + RemoteFileDescriptor folderDesc, + Set handledAccountingEntries, + Set handledWorkspaceItemEntries, + WorkspaceFolder toScanFolder, + ThreddsController folderController, + SynchFolderConfiguration config, + Process ownerProcess, + ExecutorService service) throws InternalException, InternalErrorException{ + + log.debug("Checking remote content for {}. Remote Absolute Path is {} ",toScanFolder.getPath(),folderDesc.getAbsolutePath()); + Set handledRemoteElements=new HashSet(); + +// String relativePath=toScanFolder.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + + + if(!folderDesc.isDirectory()) throw new InternalException("Remote Descriptor "+folderDesc.getAbsolutePath()+" Is not a directory. "); + for(String child:folderDesc.getChildren()) { + // skip if already handled with accounting + if(handledAccountingEntries.contains(child)) + log.debug("Skipping remote child {} because already handled with accouting", child); + // skip if already handled with local items + else if(handledWorkspaceItemEntries.contains(child)) + log.debug("Skipping remote child {} because already handled with respective item",child); + else { + RemoteFileDescriptor childDesc=folderController.getFileDescriptor(child); + if(childDesc.isDirectory()) { + handledRemoteElements.add(child); + }else if (config.matchesFilter(child)){ + log.debug("Child {} matches filter..."); + handledRemoteElements.add(child); + if(service!=null) { + service.execute(new SynchronizationThread(new TransferFromThreddsRequest(ownerProcess, null, toScanFolder, child))); + log.debug("Submitted IMPORT request number {} ",ownerProcess.getStatus().getQueuedTransfers().incrementAndGet()); + } + // import if matching + }else log.debug("Skipping not matching remote {} ",child); + // skip if doesn't match filter or isn't folder + } + } + return handledRemoteElements; + } + + + static void initProperties(WorkspaceItem toInit, String remotePath, String filter, String targetToken, + String catalogName,Boolean validateMeta, String rootFolderId) throws InternalErrorException { + + Map toSetProperties=toInit.getProperties().getProperties(); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.TBS,"true"); + + + initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_TIME,0l+""); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.LAST_UPDATE_STATUS,StepReport.Status.OK+""); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS,SynchronizationStatus.UP_TO_DATE+""); + + if(toInit.isFolder()) { + initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.TARGET_TOKEN,targetToken); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.RELATED_CATALOG,catalogName); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.VALIDATE_METADATA,validateMeta+""); + initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId); + }else { + initIfMissing(toSetProperties,Constants.WorkspaceProperties.METADATA_UUID,null); + } + toInit.getProperties().addProperties(toSetProperties); + } + + + private static void initIfMissing(Map current,String key,String defaultValue) { + if(!current.containsKey(key)|| + current.get(key)==null|| + current.get(key).equals("null")) current.put(key, defaultValue); + } + + + static boolean isConfigured(WorkspaceItem toCheck) throws InternalErrorException { + return isConfigured(toCheck.getProperties().getProperties()); + } + static boolean isConfigured(Map toCheckProperties) { + return (toCheckProperties.containsKey(Constants.WorkspaceProperties.TBS)&&toCheckProperties.get(Constants.WorkspaceProperties.TBS)!=null); + } + + static SynchronizationStatus getStatusAgainstRemote(WorkspaceItem item, Set existingRemote, ThreddsController remoteFolderController,Date lastUpdateRoutine) throws NumberFormatException, InternalErrorException, RemoteFileNotFoundException { + String itemName=item.getName(); + SynchronizationStatus status=SynchronizationStatus.OUTDATED_REMOTE; + if(existingRemote.contains(itemName)) { + RemoteFileDescriptor desc=remoteFolderController.getFileDescriptor(itemName); + Date remoteDate=new Date(desc.getLastUpdate()); + Date localDate=item.getLastModificationTime().getTime(); + Date lastUpdate=safelyGetLastUpdate(item); + + if(localDate.equals(lastUpdate)) { + //LAST MODIFCATION WAS FROM SYNCHRONIZATION + if(remoteDate.after(lastUpdate)) status=SynchronizationStatus.OUTDATED_WS; + else status=SynchronizationStatus.UP_TO_DATE; + }else + if(remoteDate.before(localDate)) { // REMOTE OLDER THAN LOCAL + if(isModifiedAfter(item,lastUpdateRoutine)) status=SynchronizationStatus.OUTDATED_REMOTE; // IT's been locally modified from last routine + else status=SynchronizationStatus.UP_TO_DATE; + } + else if(remoteDate.after(localDate)) { // REMOTE NEWER &.. + if (remoteDate.equals(lastUpdate))status =SynchronizationStatus.UP_TO_DATE; // REMOTE DATE == LAST UPDATE ROUTINE -> UP TO DATE + else if (remoteDate.before(lastUpdate))status =SynchronizationStatus.OUTDATED_REMOTE; // REMOTE DATE < LAST UPDATE -> transfer to thredds, last update was faulty + else status=SynchronizationStatus.OUTDATED_WS; // REMOTE DATE != LAST UPDATE -> import from thredds + } + } + return status; + } + + + // + // /** + // * + // * @return max date between creation time, last modification time && LAST-UPDATE-PROP + // * @throws InternalErrorException + // * @throws NumberFormatException + // */ + // static Date getMaxLastUpdate(WorkspaceItem item) throws NumberFormatException, InternalErrorException { + // return new Date(Long.max(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME)),item.getLastModificationTime().getTimeInMillis())); + // } + + static Date safelyGetLastUpdate(WorkspaceItem item) throws InternalErrorException { + try { + return new Date(Long.parseLong(item.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_TIME))); + }catch(NumberFormatException e) { + log.debug("Unable to get last update time for {} ",item.getName(),e); + return new Date(0l); + } + } + + public static boolean isModifiedAfter(WorkspaceItem item,Date fromDate) throws InternalErrorException { + for(AccountingEntry entry:item.getAccounting()) { + if(entry.getDate().getTime().after(fromDate)) { + switch(entry.getEntryType()) { + case PASTE: + case CREATE: + case RESTORE: + case UPDATE: + case ADD: return true; + } + } + } + return false; + } + + + static void cleanItem(WorkspaceItem item) throws InternalErrorException { + Properties props=item.getProperties(); + if(props.hasProperty(Constants.WorkspaceProperties.TBS)) { + if(item.isFolder()) { + props.addProperties(Constants.cleanedFolderPropertiesMap); + for(WorkspaceItem child : ((WorkspaceFolder)item).getChildren()) + cleanItem(child); + }else props.addProperties(Constants.cleanedItemPropertiesMap); + } + } + + static void setLastUpdateTime(WorkspaceFolder folder,long toSetTime) throws InternalErrorException { + StepReport.Status currentWSStatus=StepReport.Status.valueOf(folder.getProperties().getPropertyValue(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)); + + if(currentWSStatus.equals(StepReport.Status.OK)) + folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toSetTime+"")); + + for(WorkspaceItem item:folder.getChildren()) + if(item.isFolder()) setLastUpdateTime((WorkspaceFolder) item, toSetTime); + + } + + public static SynchFolderConfiguration loadConfiguration(WorkspaceItem item) throws InternalErrorException { + if(item.isFolder()) { + Properties props=item.getProperties(); + SynchFolderConfiguration config=new SynchFolderConfiguration(); + config.setFilter(props.getPropertyValue(Constants.WorkspaceProperties.SYNCH_FILTER)); + config.setRemotePath(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH)); + config.setRemotePersistence(props.getPropertyValue(Constants.WorkspaceProperties.REMOTE_PERSISTENCE)); + config.setTargetToken(props.getPropertyValue(Constants.WorkspaceProperties.TARGET_TOKEN)); + config.setToCreateCatalogName(props.getPropertyValue(Constants.WorkspaceProperties.RELATED_CATALOG)); + config.setValidateMetadata(Boolean.parseBoolean(props.getPropertyValue(Constants.WorkspaceProperties.VALIDATE_METADATA))); + config.setRootFolderId(props.getPropertyValue(Constants.WorkspaceProperties.ROOT_FOLDER_ID)); + return config; + }else return loadConfiguration(item.getParent()); + } + + static void resetStatus(WorkspaceItem item) throws InternalErrorException { + if(item.isFolder()) { + for(WorkspaceItem child: ((WorkspaceFolder)item).getChildren()) + resetStatus(child); + } + Map props=item.getProperties().getProperties(); + if(props.containsKey(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)) { + props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, StepReport.Status.OK+""); + item.getProperties().addProperties(props); + } + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java new file mode 100644 index 0000000..e82f29b --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/DeleteRemoteRequest.java @@ -0,0 +1,18 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.usecases.ws.thredds.engine.impl.Process; + +import lombok.Data; + +@Data +public class DeleteRemoteRequest extends SynchronizationRequest { + + private String toRemoveName; + + public DeleteRemoteRequest(Process process,WorkspaceFolder location,String name) { + super(process,location); + this.toRemoveName=name; + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessIdProvider.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessIdProvider.java new file mode 100644 index 0000000..c287288 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessIdProvider.java @@ -0,0 +1,30 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +public class ProcessIdProvider { + + public static ProcessIdProvider instance=new ProcessIdProvider(); + + private static final InheritableThreadLocal threadProcessId= + new InheritableThreadLocal() { + @Override + protected String initialValue() { + return null; + } + }; + + private ProcessIdProvider() { + + } + + public String get() { + return threadProcessId.get(); + } + + public void set(String processId) { + threadProcessId.set(processId); + } + + public void reset() { + threadProcessId.remove(); + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java new file mode 100644 index 0000000..bcb63a7 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/ProcessInitializationThread.java @@ -0,0 +1,59 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +import java.util.concurrent.ExecutorService; + +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.usecases.ws.thredds.engine.impl.Process; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; + +import lombok.Data; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Data +@RequiredArgsConstructor +@Slf4j +public class ProcessInitializationThread implements Runnable { + + @NonNull + private Process theProcess; + @NonNull + private ExecutorService service; + + @Override + public void run() { + ProcessDescriptor descriptor=theProcess.getDescriptor(); + ProcessIdProvider.instance.set(descriptor.getProcessId()); + log.info("Initialization of process {} ",descriptor); + theProcess.getStatus().setCurrentMessage("Gathering synchronization information..."); + try { + + WorkspaceFolderManager manager=new WorkspaceFolderManager(descriptor.getFolderId()); + + log.debug("Updateing synchronization status.."); + manager.check(true); + log.debug("Launching requests..."); + theProcess.launch(service); + + }catch(WorkspaceInteractionException e) { + log.error("Unable to proceed..",e); + theProcess.cancel(); + } catch (InternalException e) { + log.error("Unable to proceed..",e); + theProcess.cancel(); + } catch (InternalErrorException e) { + log.error("Unable to proceed..",e); + theProcess.cancel(); + }catch(Throwable t) { + log.error("Unexpected Error : ",t); + theProcess.cancel(); + }finally { + ProcessIdProvider.instance.reset(); + } + + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java new file mode 100644 index 0000000..f277ca0 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/RequestLogger.java @@ -0,0 +1,113 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +import java.io.BufferedWriter; +import java.io.FileWriter; + +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.usecases.ws.thredds.Constants; +import org.gcube.usecases.ws.thredds.SyncEngine; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RequestLogger { + + + private static RequestLogger instance=null; + + @Synchronized + public static RequestLogger get() { + if(instance==null) { + instance=new RequestLogger(); + } + return instance; + } + + + + BufferedWriter writer = null; + + private RequestLogger() { + SyncEngine engine=SyncEngine.get(); + if(engine.isRequestLoggerEnabled()) { + String path=engine.getRequestLoggerPath(); + log.info("Initializing Request Logger to path {} ",path); + try{ + writer = new BufferedWriter(new FileWriter(path, false)); + }catch(Throwable t) { + log.warn("Unable to initialize Writer on path {} ",path,t); + } + } + } + + + + public void log(SynchronizationRequest request) { + if(writer!=null) { + String logString="INVALID"; + if(request instanceof DeleteRemoteRequest) { + DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) request; + logString=String.format("DELETE %s from %s (WS-Path: %s)", deleteRequest.getToRemoveName(),getRemotePath(deleteRequest.getLocation()),getWSPath(deleteRequest.getLocation())); + }else if(request instanceof TransferToThreddsRequest) { + TransferToThreddsRequest transferRequest=(TransferToThreddsRequest) request; + logString=String.format("EXPORT %s to %s from %s", getName(transferRequest.getToTransfer()),getRemotePath(transferRequest.getToTransfer()),getWSPath(transferRequest.getToTransfer())); + }else if(request instanceof TransferFromThreddsRequest) { + TransferFromThreddsRequest transferRequest=(TransferFromThreddsRequest)request; + if(transferRequest.getTargetItem()!=null) + logString=String.format("UPDATE LOCAL %s in %s from %s ", getName(transferRequest.getTargetItem()),getWSPath(transferRequest.getLocation()),getRemotePath(transferRequest.getLocation())); + else + logString=String.format("IMPORT LOCAL %s in %s from %s ", transferRequest.getRemoteFilename(),getWSPath(transferRequest.getLocation()),getRemotePath(transferRequest.getLocation())); + } + try { + writer.write(logString+"\n"); + }catch(Throwable t) { + log.warn("Exception wile trying to write log",t); + } + } + } + + + public void close() { + if(writer!=null) { + try { + writer.flush(); + writer.close(); + }catch(Throwable t) { + log.warn("Unable to close writer ",t); + } + } + } + + + private static final String getRemotePath(WorkspaceItem item) { + try{ + if(item.isFolder()) return item.getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + else return getRemotePath(item.getParent()); + }catch(InternalErrorException e) { + log.warn("Unable to get Remote Path ",e); + return "N/A"; + } + } + + + private static final String getName(WorkspaceItem item) { + try { + return item.getName(); + }catch(InternalErrorException e) { + log.warn("Unable to get name ",e); + return "N/A"; + } + } + + private static final String getWSPath(WorkspaceItem item) { + try{ + if(item.isFolder()) return item.getPath(); + else return getRemotePath(item.getParent()); + }catch(InternalErrorException e) { + log.warn("Unable to get WS Path ",e); + return "N/A"; + } + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java new file mode 100644 index 0000000..40c9d16 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationRequest.java @@ -0,0 +1,16 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.usecases.ws.thredds.engine.impl.Process; + +import lombok.Data; +import lombok.NonNull; + +@Data +public abstract class SynchronizationRequest { + + @NonNull + private Process process; + @NonNull + private WorkspaceFolder location; +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java new file mode 100644 index 0000000..8bd2dd1 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/SynchronizationThread.java @@ -0,0 +1,289 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.data.transfer.library.TransferResult; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.ExecutionReport; +import org.gcube.data.transfer.model.ExecutionReport.ExecutionReportFlag; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.data.transfer.model.RemoteFileDescriptor; +import org.gcube.data.transfer.model.plugins.thredds.DataSet; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.spatial.data.sdi.interfaces.Metadata; +import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions; +import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; +import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder; +import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; +import org.gcube.usecases.ws.thredds.Constants; +import org.gcube.usecases.ws.thredds.NetUtils; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus; +import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; +import org.gcube.usecases.ws.thredds.faults.CancellationException; +import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException; +import org.gcube.usecases.ws.thredds.model.StepReport; +import org.gcube.usecases.ws.thredds.model.StepReport.OperationType; +import org.gcube.usecases.ws.thredds.model.StepReport.Status; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; +import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus; + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SynchronizationThread implements Runnable { + + private SynchronizationRequest theRequest; + + public SynchronizationThread(SynchronizationRequest theRequest) { + super(); + this.theRequest = theRequest; + } + + @Override + public void run() { + ProcessIdProvider.instance.set(theRequest.getProcess().getDescriptor().getProcessId()); + log.debug("Executing Transfer request {} ",theRequest); + RequestLogger.get().log(theRequest); + String reportMessage="Never started"; + String reportItemName="Still Unknown"; + Status toSetStatus=Status.ERROR; + WorkspaceFolder parentFolder=theRequest.getLocation(); + try { + checkCancelledProcess(); + SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(theRequest.getLocation()); + ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken()); + + if(theRequest instanceof TransferToThreddsRequest) { + TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest; + WorkspaceItem item=request.getToTransfer(); + + //look for metadata in same folder + String itemName=item.getName(); + reportItemName=itemName; + String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml"; + WorkspaceItem metadataItem=getFileByName(item.getParent(),false,toLookMetadataName); + + // if not present, generate with sis/geotk + Destination toSetDestination=new Destination(); + toSetDestination.setCreateSubfolders(true); + toSetDestination.setDestinationFileName(itemName); + toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE); + toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); + toSetDestination.setPersistenceId(synchConfig.getRemotePersistence()); + + //NB ITEM IS SUPPOSED TO HAVE REMOTE PATH + String fileLocation=request.getLocation().getProperties().getPropertyValue(Constants.WorkspaceProperties.REMOTE_PATH); + toSetDestination.setSubFolder(fileLocation); + + + checkCancelledProcess(); + + + Set invocations=null; + if(metadataItem==null) { + log.debug("Metadata not found, asking SIS/GEOTK for generation.."); + invocations=Collections.singleton(new PluginInvocation(Constants.SIS_PLUGIN_ID)); + } + log.info("Transferring to {} with invocations {} ",toSetDestination,invocations); + + ThreddsInfo info=controller.getThreddsInfo(); + + DataSet dataset=info.getDataSetFromLocation(info.getLocalBasePath()+"/"+fileLocation); + + // ThreddsCatalog catalog=controller.getCatalog(); + + checkCancelledProcess(); + + TransferResult result=controller.transferFile(toSetDestination, item.getPublicLink(false), invocations); + + + Map toSetProperties=new HashMap(); + + + String toSetMetadataUUID=null; + + Boolean validateMetadata=synchConfig.getValidateMetadata(); + + + checkCancelledProcess(); + if(metadataItem==null) { + ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID); + if(!report.getFlag().equals(ExecutionReportFlag.SUCCESS)) throw new DataTransferPluginError("Unable to Extract Metadata for "+itemName+" Message is "+report.getMessage()); + else toSetMetadataUUID=report.getMessage(); + + }else { + MetadataReport metaReport=publishMetadata(metadataItem,info.getHostname(),itemName,dataset.getPath(),validateMetadata); + toSetMetadataUUID=metaReport.getPublishedUUID(); + } + + toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, controller.getFileDescriptor(itemName).getLastUpdate()+""); + toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID); + + toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+""); + item.getProperties().addProperties(toSetProperties); + + reportMessage="Successfully transferred and published."; + // End ws->th + } else if(theRequest instanceof TransferFromThreddsRequest) { + Map toSetProperties=new HashMap(); + TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest; + String toImportName=null; + WorkspaceItem targetItem=null; + if(importRequest.getTargetItem()==null) { + //Target Item will be created + toImportName=importRequest.getRemoteFilename(); + reportMessage="Importing remote file.."; + }else { + //Target Item already exists + toImportName=importRequest.getTargetItem().getName(); + targetItem=importRequest.getTargetItem(); + reportMessage="Updating file.."; + } + + reportItemName=toImportName; + + RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName); + + toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+""); + InputStream source=null; + try { + source=controller.getInputStream(toImportName); + if(targetItem==null) + targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source); + else + targetItem.updateItem(source); + targetItem.getProperties().addProperties(toSetProperties); + }finally { + if(source!=null) + source.close(); + } + reportMessage="File successfully imported"; + + }else if(theRequest instanceof DeleteRemoteRequest) { + DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest; + reportItemName=deleteRequest.getToRemoveName(); + + log.debug("Going to delete {} from ",reportItemName,synchConfig.getRemotePath()); + + RemoteFileDescriptor desc=controller.getFileDescriptor(reportItemName); + if(desc.isDirectory()) { + log.debug("Remote {} is directory.. Cleaning it up, first.",reportItemName); + controller.createEmptyFolder(null); + } + controller.deleteThreddsFile(reportItemName); + reportMessage="Successfully removed"; + } + log.info("Synchronization of {} successful.",reportItemName); + toSetStatus=Status.OK; + }catch(CancellationException e) { + log.debug("Process cancelled.. ",e); + reportMessage="CancelledProcess"; + toSetStatus=Status.CANCELLED; + }catch(DataTransferPluginError e) { + log.debug("Unable to extract metadata ",e); + reportMessage="Unable to extract metadata : "+e.getMessage(); + toSetStatus=Status.ERROR; + }catch(RemoteFileNotFoundException e) { + log.debug("Remote File not found ",e); + reportMessage="Remote File not found : "+e.getMessage(); + toSetStatus=Status.ERROR; + }catch(InternalErrorException e) { + log.debug("Internal generic exception ",e); + reportMessage="Internal error : "+e.getMessage(); + toSetStatus=Status.ERROR; + }catch(Throwable t) { + log.debug("Internal generic exception ",t); + reportMessage="Unexpected exception : "+t.getMessage(); + toSetStatus=Status.ERROR; + }finally { + updateParentProperty(parentFolder, toSetStatus); + submitReport(reportItemName,reportMessage,toSetStatus); + ProcessIdProvider.instance.reset(); + } + } + + + @Synchronized + private static void updateParentProperty(WorkspaceFolder folder,StepReport.Status toSetStatus) { + try { + String currentValue=folder.getProperties().getProperties().get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS); + if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null")) + folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"")); + else { + StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue); + if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus)) + folder.getProperties().addProperties(Collections.singletonMap(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"")); + } + }catch(Throwable t) { + log.warn("Unable to update folder status ",t); + } + } + + + private void checkCancelledProcess() throws CancellationException{ + if(theRequest.getProcess().getStatus(). + getStatus().equals(ProcessStatus.Status.STOPPED)) + throw new CancellationException("Process "+theRequest.getProcess().getDescriptor().getProcessId()+" has been cancelled"); + } + + + private void submitReport(String elementName,String message,StepReport.Status status) { + StepReport report=new StepReport(elementName,message,status,OperationType.WS_TO_TH,System.currentTimeMillis()); + + if(theRequest instanceof TransferToThreddsRequest) report.setOperationType(OperationType.WS_TO_TH); + else if(theRequest instanceof TransferFromThreddsRequest) report.setOperationType(OperationType.TH_TO_WS); + else if(theRequest instanceof DeleteRemoteRequest) report.setOperationType(OperationType.DELETE_REMOTE); + else throw new RuntimeException("Unknown operation request "+theRequest); + theRequest.getProcess().onStep(report); + } + + + + + private static MetadataReport publishMetadata(WorkspaceItem toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{ + File tempMetaFile=null; + try { + Metadata meta=SDIAbstractPlugin.metadata().build(); + tempMetaFile=NetUtils.download(toPublish.getPublicLink(false)); + + log.debug("Publishing metadata {} ",filename); + + MetadataPublishOptions opts=new MetadataPublishOptions( + new TemplateInvocationBuilder().threddsOnlineResources(threddsHostname, filename, publicPath).get()); + opts.setGeonetworkCategory("Datasets"); + opts.setValidate(validate); + return meta.pushMetadata(tempMetaFile, opts); + }catch(Throwable t) { + if(tempMetaFile!=null) Files.deleteIfExists(tempMetaFile.toPath()); + throw new Exception("Something went wrong while publishing metadata for "+filename+". Cause : "+t.getMessage(),t); + } + } + + + private static final WorkspaceItem getFileByName(WorkspaceFolder toLookIntoFolder,boolean recursive,String toLookForName) throws InternalErrorException { + log.debug("Looking for {} into {} [recursive {} ]",toLookForName,toLookIntoFolder.getPath()+" ID "+toLookIntoFolder.getId(),recursive); + for(WorkspaceItem item : toLookIntoFolder.getChildren()) + if(!item.isFolder()&&item.getName().equals(toLookForName)) return item; + + if(recursive) { + for(WorkspaceItem item : toLookIntoFolder.getChildren()) + if(item.isFolder()) return getFileByName((WorkspaceFolder) item, recursive, toLookForName); + } + + return null; + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java new file mode 100644 index 0000000..160cfd7 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferFromThreddsRequest.java @@ -0,0 +1,26 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.usecases.ws.thredds.engine.impl.Process; + +import lombok.Data; + +@Data +public class TransferFromThreddsRequest extends SynchronizationRequest { + + private WorkspaceItem targetItem; // can be null + + private String remoteFilename; + + public TransferFromThreddsRequest(Process process, WorkspaceItem targetItem, WorkspaceFolder containingFolder, + String remoteFilename) { + super(process,containingFolder); + this.targetItem = targetItem; + this.remoteFilename = remoteFilename; + } + + + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java new file mode 100644 index 0000000..42bbcaf --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/engine/impl/threads/TransferToThreddsRequest.java @@ -0,0 +1,23 @@ +package org.gcube.usecases.ws.thredds.engine.impl.threads; + +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.usecases.ws.thredds.engine.impl.Process; + +import lombok.Getter; +import lombok.Setter; + + +@Getter +@Setter + +public class TransferToThreddsRequest extends SynchronizationRequest { + + private WorkspaceItem toTransfer; + + public TransferToThreddsRequest(Process process,WorkspaceFolder location, WorkspaceItem toTransfer) { + super(process,location); + this.toTransfer = toTransfer; + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/CancellationException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/CancellationException.java new file mode 100644 index 0000000..ff10a0b --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/CancellationException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class CancellationException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 736538914489728352L; + + public CancellationException() { + // TODO Auto-generated constructor stub + } + + public CancellationException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public CancellationException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public CancellationException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public CancellationException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/DataTransferPluginError.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/DataTransferPluginError.java new file mode 100644 index 0000000..fae24a3 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/DataTransferPluginError.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class DataTransferPluginError extends Exception { + + /** + * + */ + private static final long serialVersionUID = 5928532367397696543L; + + public DataTransferPluginError() { + // TODO Auto-generated constructor stub + } + + public DataTransferPluginError(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public DataTransferPluginError(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public DataTransferPluginError(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public DataTransferPluginError(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/InternalException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/InternalException.java new file mode 100644 index 0000000..056d93f --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/InternalException.java @@ -0,0 +1,37 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class InternalException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 3088246150497872471L; + + public InternalException() { + super(); + // TODO Auto-generated constructor stub + } + + public InternalException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public InternalException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public InternalException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public InternalException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/LockNotOwnedException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/LockNotOwnedException.java new file mode 100644 index 0000000..beca1d9 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/LockNotOwnedException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class LockNotOwnedException extends InternalException { + + /** + * + */ + private static final long serialVersionUID = 96212116766290884L; + + public LockNotOwnedException() { + // TODO Auto-generated constructor stub + } + + public LockNotOwnedException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public LockNotOwnedException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public LockNotOwnedException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public LockNotOwnedException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/ProcessNotFoundException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/ProcessNotFoundException.java new file mode 100644 index 0000000..7da117d --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/ProcessNotFoundException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class ProcessNotFoundException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 1L; + + public ProcessNotFoundException() { + // TODO Auto-generated constructor stub + } + + public ProcessNotFoundException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public ProcessNotFoundException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public ProcessNotFoundException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public ProcessNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/RemoteFileNotFoundException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/RemoteFileNotFoundException.java new file mode 100644 index 0000000..e8655a4 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/RemoteFileNotFoundException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class RemoteFileNotFoundException extends InternalException { + + /** + * + */ + private static final long serialVersionUID = -4333525223191661141L; + + public RemoteFileNotFoundException() { + // TODO Auto-generated constructor stub + } + + public RemoteFileNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public RemoteFileNotFoundException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public RemoteFileNotFoundException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public RemoteFileNotFoundException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/UnableToLockException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/UnableToLockException.java new file mode 100644 index 0000000..dcd4846 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/UnableToLockException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class UnableToLockException extends InternalException { + + /** + * + */ + private static final long serialVersionUID = -6926321942546668032L; + + public UnableToLockException() { + // TODO Auto-generated constructor stub + } + + public UnableToLockException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public UnableToLockException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public UnableToLockException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public UnableToLockException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceFolderNotRootException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceFolderNotRootException.java new file mode 100644 index 0000000..42aa638 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceFolderNotRootException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class WorkspaceFolderNotRootException extends WorkspaceInteractionException { + + /** + * + */ + private static final long serialVersionUID = 5025376971995804122L; + + public WorkspaceFolderNotRootException() { + // TODO Auto-generated constructor stub + } + + public WorkspaceFolderNotRootException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public WorkspaceFolderNotRootException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public WorkspaceFolderNotRootException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public WorkspaceFolderNotRootException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceInteractionException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceInteractionException.java new file mode 100644 index 0000000..a09bdaa --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceInteractionException.java @@ -0,0 +1,37 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class WorkspaceInteractionException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -1121867242282602650L; + + public WorkspaceInteractionException() { + super(); + // TODO Auto-generated constructor stub + } + + public WorkspaceInteractionException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public WorkspaceInteractionException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public WorkspaceInteractionException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public WorkspaceInteractionException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceLockedException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceLockedException.java new file mode 100644 index 0000000..ac1d5e7 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceLockedException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class WorkspaceLockedException extends WorkspaceInteractionException { + + /** + * + */ + private static final long serialVersionUID = 4220255895750039163L; + + public WorkspaceLockedException() { + // TODO Auto-generated constructor stub + } + + public WorkspaceLockedException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public WorkspaceLockedException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public WorkspaceLockedException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public WorkspaceLockedException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceNotSynchedException.java b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceNotSynchedException.java new file mode 100644 index 0000000..07721e8 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/faults/WorkspaceNotSynchedException.java @@ -0,0 +1,35 @@ +package org.gcube.usecases.ws.thredds.faults; + +public class WorkspaceNotSynchedException extends WorkspaceInteractionException { + + /** + * + */ + private static final long serialVersionUID = -321145123900245553L; + + public WorkspaceNotSynchedException() { + // TODO Auto-generated constructor stub + } + + public WorkspaceNotSynchedException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public WorkspaceNotSynchedException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public WorkspaceNotSynchedException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public WorkspaceNotSynchedException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/CompletionCallback.java b/src/main/java/org/gcube/usecases/ws/thredds/model/CompletionCallback.java new file mode 100644 index 0000000..d488f24 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/CompletionCallback.java @@ -0,0 +1,9 @@ +package org.gcube.usecases.ws.thredds.model; + +import org.gcube.usecases.ws.thredds.engine.impl.Process; + +public interface CompletionCallback { + + public void onProcessCompleted(Process completedProcess); + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/StepReport.java b/src/main/java/org/gcube/usecases/ws/thredds/model/StepReport.java new file mode 100644 index 0000000..b48d62c --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/StepReport.java @@ -0,0 +1,25 @@ +package org.gcube.usecases.ws.thredds.model; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class StepReport { + + public static enum Status{ + ERROR,OK,CANCELLED + } + + + public static enum OperationType{ + WS_TO_TH,TH_TO_WS,DELETE_REMOTE + } + + private String elementName; + private String message; + private Status status; + private OperationType operationType; + private long completionTime; + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/SyncFolderDescriptor.java b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncFolderDescriptor.java new file mode 100644 index 0000000..11ae2da --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncFolderDescriptor.java @@ -0,0 +1,26 @@ +package org.gcube.usecases.ws.thredds.model; + +import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; + +import lombok.Data; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@Data +@RequiredArgsConstructor +public class SyncFolderDescriptor { + + @NonNull + private String folderId; + @NonNull + private String folderPath; + @NonNull + private SynchFolderConfiguration configuration; + @NonNull + private boolean isLocked=false; + + + private ProcessDescriptor localProcessDescriptor=null; + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationCallBack.java b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationCallBack.java new file mode 100644 index 0000000..a0aee1b --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationCallBack.java @@ -0,0 +1,10 @@ +package org.gcube.usecases.ws.thredds.model; + +import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus; + +public interface SyncOperationCallBack { + + public void onStep(ProcessStatus status, ProcessDescriptor descriptor); + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationTicket.java b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationTicket.java new file mode 100644 index 0000000..4eb0c49 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/SyncOperationTicket.java @@ -0,0 +1,5 @@ +package org.gcube.usecases.ws.thredds.model; + +public class SyncOperationTicket { + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/SynchFolderConfiguration.java b/src/main/java/org/gcube/usecases/ws/thredds/model/SynchFolderConfiguration.java new file mode 100644 index 0000000..6c2c671 --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/SynchFolderConfiguration.java @@ -0,0 +1,37 @@ +package org.gcube.usecases.ws.thredds.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@RequiredArgsConstructor +public class SynchFolderConfiguration { + + @NonNull + private String remotePath; + @NonNull + private String filter; + @NonNull + private String targetToken; + + + @NonNull + private String toCreateCatalogName; + + private String remotePersistence="thredds"; + + @NonNull + private Boolean validateMetadata=true; + + @NonNull + private String rootFolderId; + + public boolean matchesFilter(String name) { + return name.endsWith(".nc")||name.endsWith(".ncml")||name.endsWith(".asc"); + } +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/SynchronizedElementInfo.java b/src/main/java/org/gcube/usecases/ws/thredds/model/SynchronizedElementInfo.java new file mode 100644 index 0000000..6e5440d --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/SynchronizedElementInfo.java @@ -0,0 +1,10 @@ +package org.gcube.usecases.ws.thredds.model; + +public class SynchronizedElementInfo { + + public static enum SynchronizationStatus{ + UP_TO_DATE,OUTDATED_WS,OUTDATED_REMOTE + } + + +} diff --git a/src/main/java/org/gcube/usecases/ws/thredds/model/gui/CatalogBean.java b/src/main/java/org/gcube/usecases/ws/thredds/model/gui/CatalogBean.java new file mode 100644 index 0000000..8fae2fb --- /dev/null +++ b/src/main/java/org/gcube/usecases/ws/thredds/model/gui/CatalogBean.java @@ -0,0 +1,43 @@ +package org.gcube.usecases.ws.thredds.model.gui; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@AllArgsConstructor +@ToString +public class CatalogBean { + + private String name; + private String path; + private Boolean isDefault; + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((path == null) ? 0 : path.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + CatalogBean other = (CatalogBean) obj; + if (path == null) { + if (other.path != null) + return false; + } else if (!path.equals(other.path)) + return false; + return true; + } + + + +} diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..38535ae --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,11 @@ +# Root logger option +#log4j.rootLogger=INFO, SM +log4j.logger.org.gcube.usecases.ws=DEBUG,SM + + + +#SM-specific logger with dedicated appender +log4j.appender.SM=org.apache.log4j.ConsoleAppender +log4j.appender.SM.Target=System.out +log4j.appender.SM.layout=org.apache.log4j.PatternLayout +log4j.appender.SM.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..e69de29 diff --git a/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties b/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties new file mode 100644 index 0000000..ccdd3c5 --- /dev/null +++ b/src/main/resources/org/gcube/usecases/ws/thredds/configuration.properties @@ -0,0 +1,6 @@ +scanner.pool.maxSize=10 +scanner.pool.coreSize=1 +scanner.pool.idle.ms=30000 +transfers.pool.maxSize=40 +transfers.pool.coreSize=1 +transfers.pool.idle.ms=30000 \ No newline at end of file diff --git a/src/test/java/org/gcube/usecases/ws/thredds/DTSynchUseCase.java b/src/test/java/org/gcube/usecases/ws/thredds/DTSynchUseCase.java new file mode 100644 index 0000000..0675951 --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/DTSynchUseCase.java @@ -0,0 +1,78 @@ +package org.gcube.usecases.ws.thredds; + +import java.util.concurrent.Semaphore; + +import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; +import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class DTSynchUseCase { + + public static void main(String[] args) throws WorkspaceInteractionException, InternalException, ProcessNotFoundException { + TokenSetter.set("/d4science.research-infrastructures.eu"); + SyncEngine engine=SyncEngine.get(); + + String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d"; + + +// String folderId="8a6f9749-68d7-4a9a-a475-bd645050c3fd"; // sub folder for faster tests + System.out.println("Clearing configuration.. "); + + try { + engine.unsetSynchronizedFolder(folderId, false); + }catch(WorkspaceNotSynchedException e) { + + } + System.out.println("Setting configuration"); + + + SynchFolderConfiguration config=new SynchFolderConfiguration("public/netcdf/GPTest", "", + "***REMOVED***", // devVRE + "Agro",folderId); + + engine.setSynchronizedFolder(config, folderId); + + System.out.println("Invoke check... "); + engine.check(folderId, false); + + + + System.out.println("Invoke synch"); + + engine.doSync(folderId); + + Semaphore sem=new Semaphore(0); + + engine.registerCallBack(folderId, new SyncOperationCallBack() { + + @Override + public void onStep(ProcessStatus status, ProcessDescriptor descriptor) { + System.out.println("ON STEP : "+status+" "+descriptor); + System.out.println("LOG : \n"+ status.getLogBuilder().toString()); + if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release(); + } + }); + + System.out.println("Waiting for process.. "); + try { + sem.acquire(); + } catch (InterruptedException e) { + + } + + System.out.println("Done"); + + engine.check(folderId, false); + + + // INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES + engine.shutDown(); + + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java b/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java new file mode 100644 index 0000000..5121a67 --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/DTTests.java @@ -0,0 +1,67 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Collections; + +import org.apache.commons.io.IOUtils; +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.ServiceNotFoundException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.library.faults.UnreachableNodeException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.data.transfer.model.PluginInvocation; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class DTTests { + + public static void main(String[] args) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException, MalformedURLException, FileNotFoundException, IOException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { +// TestCommons.setScope(); +// String threddsHostname="thredds-d-d4s.d4science.org"; +// DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+threddsHostname+":80"); +// Destination toSetDestination=new Destination(); +// toSetDestination.setCreateSubfolders(true); +// toSetDestination.setDestinationFileName("transferTest.tst"); +// toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE); +// toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND); +// toSetDestination.setPersistenceId("thredds"); +// +// //NB ITEM IS SUPPOSED TO HAVE REMOTE PATH +// String fileLocation="WS-Tests/mySub"; +// toSetDestination.setSubFolder(fileLocation); +// +// File temp=File.createTempFile("testTransfer", "tmp"); +// IOUtils.copy(new URL("http://data-d.d4science.org/SUlDWjIxamdaUTdHcmpvdEFmcFFPOUcvbjF5VyswbXlHbWJQNStIS0N6Yz0").openStream(), new FileOutputStream(temp)); +// +// System.out.println(client.localFile(temp, +// toSetDestination,Collections.singleton(new PluginInvocation(Constants.SIS_PLUGIN_ID)))); +// + + TokenSetter.set("/gcube/devNext"); + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + String folderId=ws.getItemByPath("/Workspace/Accounting").getId(); + SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId); + System.out.println("Done"); + + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/FileAccessTests.java b/src/test/java/org/gcube/usecases/ws/thredds/FileAccessTests.java new file mode 100644 index 0000000..3192853 --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/FileAccessTests.java @@ -0,0 +1,78 @@ +package org.gcube.usecases.ws.thredds; + +import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.UnableToLockException; + +public class FileAccessTests { + + public static void main(String[] args) throws InternalException { + TestCommons.setScope(); + String processID="stupidProcess"; + String toLockFolder="devVRE"; + + final ThreddsController controller=new ThreddsController(toLockFolder, TokenSetter.getCurrentToken()); + + Runnable readCatalog=new Runnable() {@Override public void run() { try{ + controller.readThreddsFile("catalog.xml"); + + }catch(Throwable t) {System.err.println(t); }}}; + + + Runnable readInfo=new Runnable() {@Override public void run() { try{ + controller.getThreddsInfo(); +}catch(Throwable t) {System.err.println(t); }}}; + + + repeatedTimedOperation(readCatalog,"READ CATALOG"); + + repeatedTimedOperation(readInfo, "READ INFO"); + + + System.out.println("Checking lock .."); + + + + try { + controller.lockFolder(processID); + }catch(UnableToLockException e) { + System.out.println("Dirty from previous tests.. removing lock.."); + controller.deleteThreddsFile(Constants.LOCK_FILE); + System.out.println("Trying to lock again.."); + controller.lockFolder(processID); + } + + try{ + System.out.println("Reading file descriptor... "); + System.out.println(controller.getFileDescriptor(Constants.LOCK_FILE)); + System.out.println("Ok, next should fail..."); + controller.lockFolder(processID); + }catch(UnableToLockException e) { + System.out.println("Correctly failed, now we read lock.. "); + + String lockId=controller.readThreddsFile(Constants.LOCK_FILE); + if(lockId.equals(processID)) { + System.out.println("Lock owned. Going to remove it.. "); + controller.deleteThreddsFile(Constants.LOCK_FILE); + System.out.println("Now I can lock it again... "); + controller.lockFolder(processID); + System.out.println("Ok, cleaning it up.. "); + controller.deleteThreddsFile(Constants.LOCK_FILE); + }else throw new RuntimeException("LOCK ID "+lockId+" IS DIFFERENT then expected "+processID); + } + + + } + + + private static void repeatedTimedOperation(Runnable op,String title) { + System.out.println(title); + for(int i=0;i<10;i++) { + long startTime=System.currentTimeMillis(); + op.run(); + System.out.println("["+i+"] in "+(System.currentTimeMillis()-startTime)); + } + + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java b/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java new file mode 100644 index 0000000..469c9db --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/GetWSSynchSpace.java @@ -0,0 +1,46 @@ +package org.gcube.usecases.ws.thredds; + +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.homelibrary.home.workspace.folder.FolderItem; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class GetWSSynchSpace { + + public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { + TokenSetter.set("/gcube/preprod/preVRE"); + SyncEngine engine=SyncEngine.get(); + + String folderId=TestCommons.getWSIdByPath("/Workspace/Thredds main catalog"); + + System.out.println(folderId); + + + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + WorkspaceFolder folder=(WorkspaceFolder) ws.getItem(folderId); + + SynchFolderConfiguration config=new SynchFolderConfiguration(); + System.out.println("Total size : "+computeLength(folder, config)); + } + + + private static final long computeLength(WorkspaceItem item,SynchFolderConfiguration config) throws InternalErrorException { + long toReturn=0l; + if(item.isFolder()) { + for(WorkspaceItem child:((WorkspaceFolder)item).getChildren()) { + if(item.isFolder()||config.matchesFilter(item.getName())) + toReturn=toReturn+computeLength(child, config); + } + }else toReturn=toReturn+((FolderItem)item).getLength(); + return toReturn; + } + + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java b/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java new file mode 100644 index 0000000..a3e57ee --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/PublicLinkIssueTest.java @@ -0,0 +1,103 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.File; +import java.util.Map; +import java.util.Map.Entry; + +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Properties; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.data.transfer.library.DataTransferClient; +import org.gcube.data.transfer.library.faults.DestinationNotSetException; +import org.gcube.data.transfer.library.faults.FailedTransferException; +import org.gcube.data.transfer.library.faults.InitializationException; +import org.gcube.data.transfer.library.faults.InvalidDestinationException; +import org.gcube.data.transfer.library.faults.InvalidSourceException; +import org.gcube.data.transfer.library.faults.SourceNotSetException; +import org.gcube.data.transfer.model.Destination; +import org.gcube.data.transfer.model.DestinationClashPolicy; +import org.gcube.spatial.data.sdi.interfaces.Metadata; +import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions; +import org.gcube.spatial.data.sdi.model.metadata.MetadataReport; +import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder; +import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class PublicLinkIssueTest { + + + public static void main (String[] args) throws ItemNotFoundException, InternalErrorException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException { + TokenSetter.set("/d4science.research-infrastructures.eu"); + + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + System.out.println(ws.getItemByPath("/Workspace/ArgoNetCDF/Practical_salinity/Practical_salinity_code_30_date_2000_6.nc").getPublicLink(false)); + + String threddsHostName = Commons.getThreddsHost(); + + DataTransferClient client=Commons.getDTClient(threddsHostName); + Destination dest=new Destination("thredds", "public/netcdf/someWhere", "myTest.txt", true, DestinationClashPolicy.REWRITE, DestinationClashPolicy.APPEND); + + + + + + client.httpSource("http://data.d4science.org/V2drR2gxSFRTQlpLVC9nakozL29QcDdPR2U5UEVHYWRHbWJQNStIS0N6Yz0", dest); + + + scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d")); + System.out.println("OK FIRST ..."); + try { + Metadata meta=SDIAbstractPlugin.metadata().build(); + + + + MetadataPublishOptions opts=new MetadataPublishOptions(new TemplateInvocationBuilder().threddsOnlineResources(threddsHostName, "myMeta", "testCatalog").get()); + opts.setGeonetworkCategory("Datasets"); + MetadataReport report=meta.pushMetadata(new File("/home/fabio/Desktop/meta.xml"), opts); + }catch(Exception e) { + e.printStackTrace(); + } + System.out.println("CHECKING AGAIN"); + scanForPrint((WorkspaceFolder) ws.getItem("a8cd78d3-69e8-4d02-ac90-681b2d16d84d")); + + + } + + public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException { + System.out.println("Folder "+folder.getPath()); + printProperties(folder.getProperties()); + SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "",""); + for(WorkspaceItem item:folder.getChildren()) + if(!item.isFolder()&&config.matchesFilter(item.getName())) { +// System.out.println("ITEM "+item.getPath()); + printProperties(item.getProperties()); + } + for(WorkspaceItem item:folder.getChildren()) + if(item.isFolder())scanForPrint((WorkspaceFolder) item); + } + + + public static void printProperties(Properties prop) throws InternalErrorException { + Map map=prop.getProperties(); +// System.out.print("Properties : .."); + for(Entry entry:map.entrySet()) { + if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)|| + entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)|| + entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) { +// if(true) { + if(entry.getValue()==null) System.out.print(entry.getKey()+" is null;"); + else System.out.print(entry.getKey()+" = "+entry.getValue()+";"); + } + } +// System.out.println(); + } + + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java b/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java new file mode 100644 index 0000000..871383e --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/TestCommons.java @@ -0,0 +1,113 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.HashMap; +import java.util.Map; + +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +public class TestCommons { + + @Getter + @AllArgsConstructor + static class TestSet{ + String label; + String scope; + String folderId; + String remotePath; + String targetToken; + String toCreateCatalogName; + + + public String getFolderId() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + if(folderId.startsWith("/")) + return getWSIdByPath(folderId); + else return folderId; + } + + } + + private static Map configs=new HashMap<>(); + + private static String toUseConfig="pre"; + + + static { + + configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","","")); + +// folderName="WS-Tests"; + + configs.put("default", new TestSet("Default Tests","/gcube/devsec/devVRE","/Workspace/ThreddsDev","public/netcdf","***REMOVED***","main")); + + + configs.put("pre", new TestSet("Default Tests","/gcube/preprod/preVRE","/Workspace/CMEMS","public/netcdf/CMEMS","***REMOVED***","main")); + + } + + + public static String getWSIdByPath(String path) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException { + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + return ws.getItemByPath(path).getId(); + } + + + public static void setScope() { + TokenSetter.set(configs.get(toUseConfig).getScope()); + } + + + public static WorkspaceFolder getTestFolder() throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, IOException { + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + try{ + return (WorkspaceFolder) ws.getItem(configs.get(toUseConfig).getFolderId()); + }catch(Throwable t) { + // try to use path + return (WorkspaceFolder) ws.getItemByPath(configs.get(toUseConfig).getFolderId()); + } + + + +// WorkspaceFolder folder=null; +// try{ +// folder=ws.getRoot().createFolder(folderName+"2", "test purposes"); +// }catch(ClassCastException e) { +// folder=(WorkspaceFolder) ws.getItemByPath("/Workspace/"+folderName+"2"); +// } +// +// String datasetUrl="https://thredds-d-d4s.d4science.org/thredds/fileServer/public/netcdf/test%20by%20Francesco/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc"; +// try { +// folder.createExternalFileItem("dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc", "nc test file", "application/x-netcdf", new URL(datasetUrl).openStream()); +// }catch(Exception e) { +// // file already existing.. +// } +// return folder; + } + + + public static ThreddsController getThreddsController() throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + SynchFolderConfiguration config=getSynchConfig(); + return new ThreddsController(config.getRemotePath(), config.getTargetToken()); + } + + public static SynchFolderConfiguration getSynchConfig() throws WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + TestSet set=configs.get(toUseConfig); + return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId()); + } +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/ThreddsCatalogTests.java b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsCatalogTests.java new file mode 100644 index 0000000..17f4e9e --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsCatalogTests.java @@ -0,0 +1,71 @@ +package org.gcube.usecases.ws.thredds; + +import org.gcube.data.transfer.library.utils.ScopeUtils; +import org.gcube.data.transfer.model.plugins.thredds.DataSetScan; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; +import org.gcube.usecases.ws.thredds.faults.InternalException; + +public class ThreddsCatalogTests { + + public static void main(String[] args) throws InternalException { + TokenSetter.set("/gcube/devsec"); + + System.out.println("Current scope is "+ScopeUtils.getCurrentScope()); + + // SyncEngine engine=SyncEngine.get(); + // + // String[] toCheckTokens=new String[] { + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // "***REMOVED***", + // }; + // + // + // for(String token:toCheckTokens) { + // System.out.println("Checking catalog for token "+token); + // for(CatalogBean bean:engine.getAvailableCatalogsByToken(token)) + // System.out.println(bean.getName()+" in "+bean.getPath()+" Default : "+bean.getIsDefault()); + // + // System.out.println("************************************"); + // System.out.println(); + // } + // + + + ThreddsController controller=new ThreddsController("",TokenSetter.getCurrentToken()); + + ThreddsInfo info=controller.getThreddsInfo(); + String [] paths=new String[] { + "/data/content/thredds/newer", + "/data/content/thredds/public", + "/data/content/thredds/public/netcdf", + "/data/content/thredds/public/netcdf/GPTest", + }; + + for(String path:paths) { + if(info.getCatalogByFittingLocation(path)!=null) { + DataSetScan ds=(DataSetScan) info.getDataSetFromLocation(path); + if(ds==null) + System.out.println("Catalog for "+path+"\t is null."); + else System.out.println("Catalog for "+path+"\t : "+ds.getLocation()+"\t "+ds.getName()); + }else System.out.println("No catalog for path "+path); + } + + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java new file mode 100644 index 0000000..9aed7fd --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/ThreddsTests.java @@ -0,0 +1,30 @@ +package org.gcube.usecases.ws.thredds; + +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo; +import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class ThreddsTests { + + public static void main(String[] args) throws InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException { + TestCommons.setScope(); + + SynchFolderConfiguration folderConfig=TestCommons.getSynchConfig(); + ThreddsController controller=TestCommons.getThreddsController(); + + System.out.println("Getting thredds info..."); + + ThreddsInfo info=controller.getThreddsInfo(); + + System.out.println("INFO "+info); + + System.out.println(info.getCatalogByFittingLocation(info.getLocalBasePath()+"/"+folderConfig.getRemotePath())); + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java b/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java new file mode 100644 index 0000000..8b1b0ed --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/WSTimes.java @@ -0,0 +1,40 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.IOException; +import java.net.MalformedURLException; + +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; + +public class WSTimes { + + public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, InternalErrorException, HomeNotFoundException, UserNotFoundException, IOException { + TestCommons.setScope(); + WorkspaceFolder folder=TestCommons.getTestFolder(); + for(WorkspaceItem item : folder.getChildren()) { + printDates(item); + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + + } + item.getProperties().addProperty("Fake prop", "fake value"); + printDates(item); + } + } + + + public static final void printDates(WorkspaceItem item) throws InternalErrorException { + System.out.println("ITEM : "+item.getName()); + System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getCreationTime().getTime())); + System.out.println("Creation Date : "+Constants.DATE_FORMAT.format(item.getLastModificationTime().getTime())); + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java new file mode 100644 index 0000000..77f96fc --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceAccounting.java @@ -0,0 +1,75 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.IOException; +import java.util.Date; + +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils; + +public class WorkspaceAccounting { + + public static void main(String[] args) throws InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, IOException, WorkspaceFolderNotFoundException, HomeNotFoundException, UserNotFoundException, NumberFormatException, ItemNotFoundException { + TestCommons.setScope(); + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + + +// WorkspaceFolder folder=ws.getRoot().createFolder("Accounting", "test purposes"); +// +// +// WorkspaceFolder subFolder=folder.createFolder("SubFolder", "Will be removed"); +// +// +// ExternalFile toBeRemovedFile=folder.createExternalFileItem("The file", "file to be removed", "application/xml", File.createTempFile("tmp", ".tmp")); +// +// subFolder.remove(); +// toBeRemovedFile.remove(); +// +// +// for(AccountingEntry entry:folder.getAccounting()) { +// try { +// +// +// Date eventTime=entry.getDate().getTime(); +// String toDeleteRemote=null; +// switch(entry.getEntryType()) { +// case REMOVAL:{ +// AccountingEntryRemoval removalEntry=(AccountingEntryRemoval) entry; +// System.out.println(removalEntry.getItemName() +"REMOVED. FolderItemType "+removalEntry.getFolderItemType()+" ItemType "+removalEntry.getItemType()); +// +// +// break; +// } +// case RENAMING:{ +// AccountingEntryRenaming renamingEntry=(AccountingEntryRenaming) entry; +// toDeleteRemote=renamingEntry.getOldItemName(); +// +// } +// case CUT:{ +// AccountingEntryCut cut = (AccountingEntryCut) entry; +// toDeleteRemote=cut.getItemName(); +// break; +// } +// } +// +// }catch(Throwable t) { +// t.printStackTrace(); +// } +// } +// +// } + + + + System.out.println(WorkspaceUtils.isModifiedAfter(ws.getItemByPath(TestCommons.getTestFolder().getPath()+"/mySub/dissolved_oxygen_annual_5deg_ENVIRONMENT_BIOTA_.nc"), new Date(0l))); + + + } +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java new file mode 100644 index 0000000..b655557 --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceCleanup.java @@ -0,0 +1,28 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.IOException; +import java.net.MalformedURLException; + +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; + +public class WorkspaceCleanup { + + public static void main(String[] args) throws WorkspaceInteractionException, InternalException, InternalErrorException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, MalformedURLException, HomeNotFoundException, UserNotFoundException, IOException { + TestCommons.setScope(); + WorkspaceFolder folder=TestCommons.getTestFolder(); + + SyncEngine engine=SyncEngine.get(); + engine.forceUnlock(folder.getId()); + engine.unsetSynchronizedFolder(folder.getId(), false); + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java new file mode 100644 index 0000000..2a92729 --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceLock.java @@ -0,0 +1,62 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.Collections; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class WorkspaceLock { + + public static void main(String[] args) throws WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, InternalErrorException, HomeNotFoundException, UserNotFoundException, WorkspaceInteractionException, InternalException, MalformedURLException, IOException { + TestCommons.setScope(); + WorkspaceFolder folder=TestCommons.getTestFolder(); + WorkspaceFolderManager manager=new WorkspaceFolderManager(folder.getId()); + + String processID="mytest"; + try { + System.out.println("Trying to cleanup, first.. "); + folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null)); + System.out.println("FOLDER PROPERTIES : "+folder.getProperties().getProperties()); + }catch(Throwable t) { + System.err.println("Dismiss error : "); + t.printStackTrace(); + } + + + +// manager.configure(new SynchFolderConfiguration("mySynchedCatalog","*.nc,*.ncml,*.asc",SecurityTokenProvider.instance.get())); + + System.out.println("Is locked : "+manager.isLocked()); + System.out.println("locking ... "); + manager.lock(processID); + + try { + manager.dismiss(false); + System.err.println("It should have raised locked exception"); + }catch(Exception e){ + System.out.println("Ok check lock on dismiss "); + } + + manager.unlock(processID); + + manager.dismiss(false); + + + System.out.println("This should be false : "+manager.isSynched()); + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java new file mode 100644 index 0000000..737e094 --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceProperties.java @@ -0,0 +1,93 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.Map; +import java.util.Map.Entry; + +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Properties; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class WorkspaceProperties { + + public static void main(String[] args) throws WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, ItemNotFoundException, MalformedURLException, IOException, WorkspaceInteractionException, InternalException { + + TestCommons.setScope(); + WorkspaceFolder folder=TestCommons.getTestFolder(); + + + SyncEngine.get().check(folder.getId(), true); + scanForPrint(folder); + +// SyncEngine.get().shutDown(); +// for(Workspace) +// printProperties(folder.getProperties()); +// +// System.out.println("Has property : "+folder.getProperties().hasProperty(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS)); +// printProperties(folder.getProperties()); +// +// +// System.out.println("Setting property.. "); +// folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, "true")); +// +// System.out.println("Has property : "+folder.getProperties().hasProperty(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS)); +// printProperties(folder.getProperties()); +// +// +// System.out.println("Removing (setting it null) "); +// folder.getProperties().addProperties(Collections.singletonMap(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS, null)); +// +// System.out.println("Has property : "+folder.getProperties().hasProperty(org.gcube.usecases.ws.thredds.Constants.WorkspaceProperties.TBS)); +// printProperties(folder.getProperties()); +// +// String folderId= +// +// WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId); +// +// manager.configure(new SynchFolderConfiguration("myRemoteFolder","thredds","*.nc,*.ncml,*.asc")); +// +// manager.dismiss(false); + + } + + public static void scanForPrint(WorkspaceFolder folder) throws InternalErrorException { + System.out.println("Folder "+folder.getPath()); + printProperties(folder.getProperties()); + SynchFolderConfiguration config=new SynchFolderConfiguration("", "", "", "",""); + for(WorkspaceItem item:folder.getChildren()) + if(!item.isFolder()&&config.matchesFilter(item.getName())) { + System.out.println("ITEM "+item.getPath()); + printProperties(item.getProperties()); + } + for(WorkspaceItem item:folder.getChildren()) + if(item.isFolder())scanForPrint((WorkspaceFolder) item); + } + + + public static void printProperties(Properties prop) throws InternalErrorException { + Map map=prop.getProperties(); + System.out.print("Properties : .."); + for(Entry entry:map.entrySet()) { +// if(entry.getKey().equals(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)|| +// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)|| +// entry.getKey().equals(Constants.WorkspaceProperties.LAST_UPDATE_TIME)) { + if(true) { + if(entry.getValue()==null) System.out.print(entry.getKey()+" is null;"); + else System.out.print(entry.getKey()+" = "+entry.getValue()+";"); + } + } + System.out.println(); + } + +} diff --git a/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java new file mode 100644 index 0000000..2f23942 --- /dev/null +++ b/src/test/java/org/gcube/usecases/ws/thredds/WorkspaceSynchronizationTest.java @@ -0,0 +1,104 @@ +package org.gcube.usecases.ws.thredds; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.concurrent.Semaphore; + +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor; +import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus; +import org.gcube.usecases.ws.thredds.faults.InternalException; +import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException; +import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException; +import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack; +import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration; + +public class WorkspaceSynchronizationTest { + + public static void main(String[] args) throws ProcessNotFoundException, InternalErrorException, WorkspaceInteractionException, InternalException, WorkspaceFolderNotFoundException, ItemNotFoundException, InsufficientPrivilegesException, ItemAlreadyExistException, HomeNotFoundException, UserNotFoundException, MalformedURLException, IOException { + + // GET ENGINE : SINGLETON INSTANCE + SyncEngine engine=SyncEngine.get(); + engine.setRequestLogger("requests.txt"); + + //TEST INFO... + TestCommons.setScope(); + WorkspaceFolder folder=TestCommons.getTestFolder(); + + // FOLDER CONFIGURATION BEAN + SynchFolderConfiguration config=TestCommons.getSynchConfig(); + +// try { +// //try to clean it up, first.. +// System.out.println("Cleaning it up.."); +// engine.unsetSynchronizedFolder(folder.getId(), false); +// }catch(WorkspaceNotSynchedException e) { +// // it was already cleared +// }catch(WorkspaceLockedException e) { +// engine.forceUnlock(folder.getId()); +// engine.unsetSynchronizedFolder(folder.getId(), false); +// } + + + + + try { + // WHEN OPENING A FOLDER, INVOKE CHECK TO UPDATE SYNCH STATUS + engine.check(folder.getId(), false); + }catch(WorkspaceNotSynchedException e) { + System.out.println("Folder not synched, configurin it.."); + engine.setSynchronizedFolder(config, folder.getId()); + engine.check(folder.getId(), false); + }catch(WorkspaceLockedException e) { + System.out.println("Workspace locked, going to force unlock.."); // MAINLY FOR TEST PURPOSES, OR WHEN SOMETHIGN GOES WRONG.. USE CAUTIOUSLY + engine.forceUnlock(folder.getId()); + engine.check(folder.getId(), false); + } + + + + + // INVOKE SYNCHRONIZATION ON FOLDER + ProcessDescriptor descriptor=engine.doSync(folder.getId()); + + System.out.println("Obtained descriptor : "+descriptor); + + Semaphore sem=new Semaphore(0); + + // REGISTER CALLBACK TO MONITOR PROGRESS + engine.registerCallBack(folder.getId(), new SyncOperationCallBack() { + + @Override + public void onStep(ProcessStatus status, ProcessDescriptor descriptor) { + System.out.println("ON STEP : "+status+" "+descriptor); + System.out.println("LOG : \n"+ status.getLogBuilder().toString()); + if(status.getStatus().equals(ProcessStatus.Status.COMPLETED)) sem.release(); + } + }); + + System.out.println("Waiting for process.. "); + try { + sem.acquire(); + } catch (InterruptedException e) { + + } + + engine.check(folder.getId(), true); + + + // INVOKE WHEN PORTAL SHUTS DOWN TO FREE RESOURCES AND STOP SYNC PROCESSES + engine.shutDown(); + + } + +}