Compare commits
1 Commits
| Author | SHA1 | Date |
|---|---|---|
 Fabio Sinibaldi
|
4cf3a730db |
|
|
@ -1,2 +0,0 @@
|
|||
/org.eclipse.jdt.core.prefs
|
||||
/org.eclipse.core.resources.prefs
|
||||
12
CHANGELOG.md
12
CHANGELOG.md
|
|
@ -2,16 +2,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
|||
|
||||
# Changelog for org.gcube.spatial.data.ws-thredds
|
||||
|
||||
## [v1.0.1]
|
||||
|
||||
Security Fixes
|
||||
Fixes [#21783]
|
||||
|
||||
## [v1.0.0]
|
||||
Integration with new IAM
|
||||
Security Fixes
|
||||
Fixes [#21537]
|
||||
|
||||
## [v0.2.6-SNAPSHOT]
|
||||
Fixes #21265
|
||||
|
||||
## [v0.2.5]
|
||||
Fixes #21265
|
||||
|
|
|
|||
49
pom.xml
49
pom.xml
|
|
@ -1,5 +1,4 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
|
|
@ -9,7 +8,7 @@
|
|||
</parent>
|
||||
<groupId>org.gcube.spatial.data</groupId>
|
||||
<artifactId>ws-thredds</artifactId>
|
||||
<version>1.0.1</version>
|
||||
<version>0.2.6-SNAPSHOT</version>
|
||||
<name>ws-thredds</name>
|
||||
<description>prototype of WS integration with data-transfer for Thredds pubblication</description>
|
||||
|
||||
|
|
@ -30,10 +29,10 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.distribution</groupId>
|
||||
<artifactId>maven-portal-bom</artifactId>
|
||||
<version>3.6.2</version>
|
||||
<version>3.6.0</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
|
@ -43,18 +42,20 @@
|
|||
<groupId>org.gcube.spatial.data</groupId>
|
||||
<artifactId>sdi-library</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,1.3.0-SNAPSHOT)</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-annotations</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>storagehub-client-library</artifactId>
|
||||
<artifactId>storagehub-client-library</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>authorization-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data.transfer</groupId>
|
||||
|
|
@ -62,14 +63,19 @@
|
|||
<version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!-- security -->
|
||||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>oidc-library</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- JSON paths -->
|
||||
<!-- <dependency> -->
|
||||
<!-- <groupId>com.fasterxml.jackson.core</groupId> -->
|
||||
<!-- <artifactId>jackson-core</artifactId> -->
|
||||
<!-- <version>[2.8.1,2.8.11]</version> -->
|
||||
<!-- </dependency> -->
|
||||
|
||||
<!-- <dependency> -->
|
||||
<!-- <groupId>com.fasterxml.jackson.core</groupId> -->
|
||||
<!-- <artifactId>jackson-databind</artifactId> -->
|
||||
<!-- <version>[2.8.1,2.8.11]</version> -->
|
||||
<!-- </dependency> -->
|
||||
|
||||
<!-- JSON paths -->
|
||||
<dependency>
|
||||
<groupId>com.jayway.jsonpath</groupId>
|
||||
<artifactId>json-path</artifactId>
|
||||
|
|
@ -79,14 +85,9 @@
|
|||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
|
|
|
|||
|
|
@ -40,31 +40,30 @@ public class Commons {
|
|||
|
||||
|
||||
public static void cleanupFolder(String toCleanPath, String destinationToken) {
|
||||
throw new RuntimeException("ILLEGAL OPERATION");
|
||||
// String toRestoreToken=TokenSetter.getCurrentToken();
|
||||
// try{
|
||||
// log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath);
|
||||
// TokenSetter.setGcubeToken(destinationToken);
|
||||
// String hostname=getThreddsHost();
|
||||
// DataTransferClient client=getDTClient(hostname);
|
||||
// File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||
// toTransfer.createNewFile();
|
||||
// Destination dest=new Destination();
|
||||
// dest.setCreateSubfolders(true);
|
||||
// dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
// dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
|
||||
// dest.setPersistenceId("thredds");
|
||||
// dest.setSubFolder("public/netcdf/"+toCleanPath);
|
||||
// log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
||||
// client.localFile(toTransfer, dest);
|
||||
// log.info("Done");
|
||||
// }catch(Exception e) {
|
||||
// log.error("Unable to delete remote folder "+toCleanPath,e);
|
||||
// throw new RuntimeException("Unable to cleanup remote folder.");
|
||||
// }finally {
|
||||
// log.debug("Resetting original token {} ",toRestoreToken);
|
||||
// TokenSetter.set(toRestoreToken);
|
||||
// }
|
||||
String toRestoreToken=TokenSetter.getCurrentToken();
|
||||
try{
|
||||
log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath);
|
||||
TokenSetter.setToken(destinationToken);
|
||||
String hostname=getThreddsHost();
|
||||
DataTransferClient client=getDTClient(hostname);
|
||||
File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||
toTransfer.createNewFile();
|
||||
Destination dest=new Destination();
|
||||
dest.setCreateSubfolders(true);
|
||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
|
||||
dest.setPersistenceId("thredds");
|
||||
dest.setSubFolder("public/netcdf/"+toCleanPath);
|
||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
||||
client.localFile(toTransfer, dest);
|
||||
log.info("Done");
|
||||
}catch(Exception e) {
|
||||
log.error("Unable to delete remote folder "+toCleanPath,e);
|
||||
throw new RuntimeException("Unable to cleanup remote folder.");
|
||||
}finally {
|
||||
log.debug("Resetting original token {} ",toRestoreToken);
|
||||
TokenSetter.set(toRestoreToken);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ public class Constants {
|
|||
// Folder
|
||||
public static final String SYNCH_FILTER="WS-SYNCH.SYNCHRONIZATION-FILTER";
|
||||
public static final String TARGET_TOKEN="WS-SYNCH.TARGET-TOKEN";
|
||||
public static final String TARGET_CONTEXT="WS-SYNCH.TARGET-CONTEXT";
|
||||
public static final String REMOTE_PATH="WS-SYNCH.REMOTE-PATH";
|
||||
public static final String REMOTE_PERSISTENCE="WS-SYNCH.REMOTE-PERSISTENCE";
|
||||
public static final String RELATED_CATALOG="WS-SYNCH.RELATED-CATALOG";
|
||||
|
|
@ -77,7 +76,6 @@ public class Constants {
|
|||
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.REMOTE_PATH, null);
|
||||
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.REMOTE_PERSISTENCE, null);
|
||||
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.TARGET_TOKEN, null);
|
||||
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.TARGET_CONTEXT, null);
|
||||
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.TBS, null);
|
||||
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, null);
|
||||
cleanedFolderPropertiesMap.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, null);
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ public class PublishFolders {
|
|||
//
|
||||
//// String folderId="a711a8d7-5e93-498f-a29c-b888d7c2e48f"; TICKET
|
||||
//
|
||||
// String publishingUserToken="..."; //fabio @NextNext
|
||||
// String publishingUserToken="***REMOVED***"; //fabio @NextNext
|
||||
//
|
||||
//
|
||||
// FolderConfiguration folderConfig=new FolderConfiguration(publishingUserToken,folderId,"GP_CASE");
|
||||
|
|
|
|||
|
|
@ -1,18 +1,14 @@
|
|||
package org.gcube.usecases.ws.thredds;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.SynchEngineImpl;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
||||
import org.gcube.usecases.ws.thredds.faults.InternalException;
|
||||
import org.gcube.usecases.ws.thredds.faults.ProcessNotFoundException;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
|
||||
import org.gcube.usecases.ws.thredds.model.ContainerType;
|
||||
import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
|
||||
|
|
@ -26,56 +22,18 @@ public interface SyncEngine {
|
|||
return SynchEngineImpl.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checked Access Method
|
||||
*/
|
||||
|
||||
public SyncFolderDescriptor check(String folderId, boolean recursively) throws WorkspaceInteractionException, InternalException;
|
||||
public void registerCallBack(String folderId,SyncOperationCallBack callback) throws ProcessNotFoundException;
|
||||
|
||||
/**
|
||||
* Checked Access Method
|
||||
*/
|
||||
public ProcessDescriptor doSync(String folderId) throws WorkspaceInteractionException, InternalException;
|
||||
public void stopSynch(String folderId) throws ProcessNotFoundException;
|
||||
|
||||
/**
|
||||
* Checked Access Method
|
||||
*/
|
||||
public void stopSynch(String folderId) throws ProcessNotFoundException, WorkspaceInteractionException, InternalException;
|
||||
|
||||
|
||||
/**
|
||||
* Checked Access Method
|
||||
*/
|
||||
public void setSynchronizedFolder(SynchFolderConfiguration config,String folderId) throws WorkspaceInteractionException, InternalException;
|
||||
|
||||
/**
|
||||
* Checked Access Method
|
||||
*/
|
||||
public void unsetSynchronizedFolder(String folderId,boolean deleteRemoteContent) throws WorkspaceInteractionException, InternalException;
|
||||
|
||||
/**
|
||||
* NB Method To Be Implemented
|
||||
*
|
||||
* @param elementId
|
||||
* @return
|
||||
*/
|
||||
public SynchronizedElementInfo getInfo(String elementId);
|
||||
|
||||
|
||||
/**
|
||||
* Checked Access Method
|
||||
* @throws WorkspaceInteractionException
|
||||
*/
|
||||
public void updateCatalogFile(String folderId, File toUpdate) throws InternalException, WorkspaceInteractionException;
|
||||
|
||||
/**
|
||||
* Checked Access Method
|
||||
*
|
||||
*
|
||||
* @param folderId
|
||||
* @throws InternalException
|
||||
* @throws WorkspaceInteractionException
|
||||
*/
|
||||
public void updateCatalogFile(String folderId, File toUpdate) throws InternalException;
|
||||
|
||||
public void forceUnlock(String folderId)throws InternalException, WorkspaceInteractionException;
|
||||
|
||||
|
|
@ -93,17 +51,8 @@ public interface SyncEngine {
|
|||
public String getRequestLoggerPath();
|
||||
|
||||
|
||||
public Set<CatalogBean> getAvailableCatalogs() throws InternalException;
|
||||
public Set<CatalogBean> getAvailableCatalogsByToken(String token) throws InternalException;
|
||||
|
||||
public SyncEngineStatusDescriptor getStatus();
|
||||
|
||||
|
||||
|
||||
public SynchFolderConfiguration getConfig(String fodlerId) throws WorkspaceInteractionException, WorkspaceNotSynchedException;
|
||||
|
||||
|
||||
|
||||
public static SynchronizedElementInfo parseInfo(Map<String,Object> itemProperties,ContainerType itemType)throws WorkspaceNotSynchedException{
|
||||
return WorkspaceUtils.getInfo(itemProperties, org.gcube.common.storagehub.client.dsl.ContainerType.valueOf(itemType.toString()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
package org.gcube.usecases.ws.thredds;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class TokenSetter {
|
||||
|
||||
private static Properties props=null;
|
||||
|
||||
static{
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static synchronized void set(String scope){
|
||||
try{
|
||||
if(props==null) {
|
||||
props=new Properties();
|
||||
try {
|
||||
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||
}
|
||||
}
|
||||
if(!props.containsKey(scope)) throw new Exception("No token found for scope : "+scope);
|
||||
SecurityTokenProvider.instance.set(props.getProperty(scope));
|
||||
}catch(Throwable e){
|
||||
log.trace("Unable to set token for scope "+scope,e);
|
||||
}
|
||||
ScopeProvider.instance.set(scope);
|
||||
}
|
||||
|
||||
|
||||
public static void setToken(String token){
|
||||
try{
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
ScopeProvider.instance.set(entry.getContext());
|
||||
SecurityTokenProvider.instance.set(token);
|
||||
}catch(Throwable t) {
|
||||
throw new RuntimeException("Unable to set token "+token,t);
|
||||
}
|
||||
}
|
||||
|
||||
public static String getCurrentToken() {
|
||||
return SecurityTokenProvider.instance.get();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package org.gcube.usecases.ws.thredds.engine;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.data.transfer.library.DataTransferClient;
|
||||
import org.gcube.data.transfer.library.TransferResult;
|
||||
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
|
||||
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
|
||||
import org.gcube.usecases.ws.thredds.Commons;
|
||||
import org.gcube.usecases.ws.thredds.NetUtils;
|
||||
import org.gcube.usecases.ws.thredds.TokenSetter;
|
||||
import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
|
||||
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PublishThread implements Runnable {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@NonNull
|
||||
private PublishRequest request;
|
||||
@NonNull
|
||||
private ConcurrentHashMap<String,PublishReport> reports;
|
||||
|
||||
|
||||
private PublishReport publishReport;
|
||||
// private Map<String,Report> reports;
|
||||
//
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
log.info("Request is {}",request);
|
||||
log.debug("Switching from {} to {}",SecurityTokenProvider.instance.get(),request.getPublishToken());
|
||||
TokenSetter.setToken(request.getPublishToken());
|
||||
log.debug("Current scope is :{}, token is {} ",ScopeUtils.getCurrentScope(),SecurityTokenProvider.instance.get());
|
||||
|
||||
Destination dest=new Destination();
|
||||
dest.setPersistenceId("thredds");
|
||||
dest.setSubFolder("public/netcdf/"+request.getCatalog());
|
||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
dest.setCreateSubfolders(true);
|
||||
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
||||
String threddsHostName;
|
||||
|
||||
|
||||
try {
|
||||
threddsHostName = Commons.getThreddsHost();
|
||||
|
||||
DataTransferClient client=Commons.getDTClient(threddsHostName);
|
||||
|
||||
|
||||
File toPublishSource=null;
|
||||
|
||||
if(request.getMode().equals(Mode.NCML)) {
|
||||
if(request.isQueue()){
|
||||
log.debug("Waiting for queue {}, expected Count {} ",request.getQueueId(),request.getQueueCount());
|
||||
waitFor(request.getQueueId(), request.getQueueCount());
|
||||
|
||||
log.debug("Loading netcdfFile ..");
|
||||
File ncmlFile=NetUtils.download(new URL(request.getSource().getUrl()));
|
||||
String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath()));
|
||||
|
||||
for(String reportId:request.getToGatherReportsId()) {
|
||||
PublishReport report=getReport(reportId);
|
||||
//file://home/gcube/julien.barde/Workspace/DataMiner/Output_Data_Sets/Ichthyop2013.nc
|
||||
|
||||
String toSetUrl="file:/"+report.getTransferResult().getRemotePath();
|
||||
toUpdateSource=toUpdateSource.replaceAll(reportId, toSetUrl);
|
||||
}
|
||||
|
||||
toPublishSource=File.createTempFile("nc_", ".ncml");
|
||||
PrintWriter out = new PrintWriter(toPublishSource);
|
||||
out.write(toUpdateSource);
|
||||
out.flush();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
TransferResult result;
|
||||
|
||||
|
||||
// TODO NB Test run without metadata publication
|
||||
// result=client.httpSource(request.getSource().getUrl(), dest);
|
||||
// publishReport=new PublishReport(false,request.getSource().getId(),result,null);
|
||||
|
||||
|
||||
if(!request.isGenerateMeta()) {
|
||||
log.debug("Transfering before publishing meta..");
|
||||
|
||||
result = toPublishSource==null?client.httpSource(request.getSource().getUrl(), dest):
|
||||
client.localFile(toPublishSource, dest);
|
||||
|
||||
//NB DECOMMENT THIS!!!!!!
|
||||
// Metadata meta=SDIAbstractPlugin.metadata().build();
|
||||
//
|
||||
// log.debug("Publishing metadata.. ");
|
||||
//
|
||||
// MetadataPublishOptions opts=new MetadataPublishOptions(new TemplateInvocationBuilder().threddsOnlineResources(threddsHostName, request.getSource().getName(), request.getCatalog()).get());
|
||||
// opts.setGeonetworkCategory("Datasets");
|
||||
// MetadataReport report=meta.pushMetadata(request.getMetadata(), opts);
|
||||
|
||||
// publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,report);
|
||||
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,new MetadataReport());
|
||||
|
||||
|
||||
}else {
|
||||
log.debug("Metadata not provided.. ");
|
||||
if(request.isQueue()&&request.getMode().equals(Mode.NC)) {
|
||||
log.debug("Dataset file is linked in ncml, skipping metadata generation");
|
||||
result=client.httpSource(request.getSource().getUrl(), dest);
|
||||
}else
|
||||
result=client.httpSource(request.getSource().getUrl(), dest,new PluginInvocation("SIS/GEOTK"));
|
||||
|
||||
|
||||
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} catch (UnreachableNodeException | ServiceNotFoundException e) {
|
||||
log.error("Unable to find Thredds. Publish scope is {} ",ScopeUtils.getCurrentScope(),e);
|
||||
} catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException
|
||||
| InvalidDestinationException | DestinationNotSetException e) {
|
||||
log.error("Unable to transfer file, ",e);
|
||||
} catch (IOException e) {
|
||||
log.error("Unable to read/ write file. ",e);
|
||||
}
|
||||
|
||||
onCompletion();
|
||||
}
|
||||
|
||||
|
||||
private void onCompletion() {
|
||||
if(publishReport==null) publishReport=new PublishReport(true, request.getSource().getId(),request.getSource().getName(), null, null);
|
||||
publishReport(publishReport);
|
||||
|
||||
if(request.getMode().equals(Mode.NC)&&(request.isQueue())) {
|
||||
alert(request.getQueueId(),request.getQueueCount());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private PublishReport getReport(String reportId) {
|
||||
return reports.get(reportId);
|
||||
}
|
||||
|
||||
|
||||
private void publishReport(PublishReport report) {
|
||||
reports.put(report.getSourceId(), report);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// static
|
||||
|
||||
|
||||
private static ConcurrentHashMap<String,Semaphore> semaphores=new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
|
||||
private static void waitFor(String queueId,Integer expected) {
|
||||
try {
|
||||
log.debug("Waiting for queue {}. Expected Count is {} ",queueId,expected);
|
||||
semaphores.getOrDefault(queueId, new Semaphore(expected*-1)).acquire();
|
||||
} catch (InterruptedException e) {
|
||||
log.debug("Queue {} is completed.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void alert(String queueId, Integer expected) {
|
||||
log.debug("Alerting queue {}. Expected count is {} ",queueId,expected);
|
||||
Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1));
|
||||
sem.release();
|
||||
log.debug(String.format("Queue %1$s alerted. Remaining : %2$s out of %3$s ",queueId,sem.availablePermits(),expected));
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
package org.gcube.usecases.ws.thredds.engine;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.gcube.usecases.ws.thredds.FolderConfiguration;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Synchronized;
|
||||
|
||||
public class TransferRequestServer {
|
||||
|
||||
@Data
|
||||
public static class Report{
|
||||
private AtomicLong requestCount=new AtomicLong(0l);
|
||||
private AtomicLong requestServed=new AtomicLong(0l);
|
||||
private ConcurrentHashMap<String,PublishReport> reports=new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
public File toFile(FolderConfiguration configuration) {
|
||||
return toFile(configuration,this);
|
||||
}
|
||||
|
||||
|
||||
private static final File toFile(FolderConfiguration config,Report report) {
|
||||
PrintWriter writer =null;
|
||||
try {
|
||||
File toReturn=File.createTempFile("tempFile", ".tmp");
|
||||
writer=new PrintWriter(toReturn);
|
||||
|
||||
writer.println("REPORT FOR WS-SYNCH");
|
||||
writer.println("Configuratiion was : "+config);
|
||||
writer.println("Submitted runs : "+report.getRequestCount());
|
||||
writer.println("Item reports : ");
|
||||
for(Entry<String,PublishReport> entry: report.getReports().entrySet()) {
|
||||
PublishReport rep=entry.getValue();
|
||||
writer.println("*********************************************************");
|
||||
if(rep.isError()) writer.println("OPERATION IS FAILED");
|
||||
writer.println("ITEM ID : "+rep.getSourceId());
|
||||
writer.println("ITEM NAME : "+rep.getSourceName());
|
||||
if(rep.getTransferResult()!=null)writer.println("Transfer report : "+rep.getTransferResult());
|
||||
if(rep.getMetadataReport()!=null)writer.println("Metadata report : "+rep.getMetadataReport());
|
||||
}
|
||||
return toReturn;
|
||||
}catch(Throwable t) {
|
||||
throw new RuntimeException(t);
|
||||
}finally {
|
||||
if(writer!=null) {
|
||||
IOUtils.closeQuietly(writer);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Report report=new Report();
|
||||
private ExecutorService service=null;
|
||||
|
||||
public TransferRequestServer() {
|
||||
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(
|
||||
100);
|
||||
service= new ThreadPoolExecutor(1, 10, 30,
|
||||
TimeUnit.SECONDS, linkedBlockingDeque,
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
|
||||
public void put(PublishRequest request){
|
||||
System.out.println("Submitting transfer "+getReport().requestCount.incrementAndGet());
|
||||
service.execute(new PublishThread(request, getReport().getReports()));
|
||||
|
||||
// service.execute(new RequestThread(baseUrl,filename,this,publishScope,toPublishMeta));
|
||||
}
|
||||
@Synchronized
|
||||
public Report getReport(){
|
||||
return report;
|
||||
}
|
||||
|
||||
|
||||
public void waitCompletion() {
|
||||
boolean running=true;
|
||||
service.shutdown();
|
||||
while(running){
|
||||
System.out.println("******************* WAITING FOR TERMINATION ***************** ");
|
||||
try{
|
||||
running=!service.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
|
||||
}catch(InterruptedException e){
|
||||
running=!service.isTerminated();
|
||||
}
|
||||
}
|
||||
System.out.println("Service is completed : "+service.isTerminated());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import javax.ws.rs.client.ClientRequestContext;
|
||||
import javax.ws.rs.client.ClientRequestFilter;
|
||||
|
||||
import org.gcube.common.calls.Call;
|
||||
import org.gcube.common.calls.Interceptors;
|
||||
import org.gcube.common.calls.Request;
|
||||
import org.gcube.data.transfer.library.utils.ScopeUtils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class CustomAuthorizationFilter implements ClientRequestFilter {
|
||||
|
||||
|
||||
@Override
|
||||
public void filter(final ClientRequestContext rc) throws IOException {
|
||||
if (ScopeUtils.getCurrentScope()!=null){
|
||||
Request requestContext = Interceptors.executeRequestChain(new Call());
|
||||
|
||||
for (Entry<String, String> entry: requestContext.getHeaders()){
|
||||
log.debug("Copying HTTP request header {} : {}",entry.getKey(),entry.getValue());
|
||||
rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl;
|
||||
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.faults.InternalException;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public abstract class GuardedMethod<T> {
|
||||
|
||||
@Getter
|
||||
private WorkspaceFolderManager manager;
|
||||
@Getter
|
||||
private SynchFolderConfiguration configuration;
|
||||
|
||||
@Getter
|
||||
private String folderId;
|
||||
@Getter
|
||||
private T result=null;
|
||||
|
||||
public GuardedMethod(String folderId) throws WorkspaceInteractionException {
|
||||
this.folderId=folderId;
|
||||
manager=new WorkspaceFolderManager(folderId);
|
||||
configuration=manager.getSynchConfiguration();
|
||||
Security.checkOperator(configuration);
|
||||
}
|
||||
|
||||
|
||||
public GuardedMethod<T> execute() throws WorkspaceInteractionException,InternalException{
|
||||
try {
|
||||
result=run();
|
||||
return this;
|
||||
}catch(WorkspaceInteractionException|InternalException e) {
|
||||
throw e;
|
||||
}catch(Throwable t) {
|
||||
log.error("Unexpected error ",t);
|
||||
throw new InternalException("Unexpected internal error", t);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected abstract T run() throws WorkspaceInteractionException,InternalException,Exception;
|
||||
|
||||
}
|
||||
|
|
@ -18,7 +18,6 @@ import org.gcube.common.storagehub.model.exceptions.StorageHubException;
|
|||
import org.gcube.data.transfer.model.RemoteFileDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.Constants;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus.Status;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.User;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationRequest;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
|
||||
|
|
@ -59,21 +58,15 @@ public class Process {
|
|||
|
||||
private CompletionCallback callback=null;
|
||||
|
||||
public Process(User operator, String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException {
|
||||
|
||||
log.debug("Created Process with id {}, operator {} ",processId,operator);
|
||||
|
||||
public Process(String folderId,CompletionCallback callback) throws WorkspaceInteractionException, InternalException {
|
||||
log.debug("Created Process with id {} ",processId);
|
||||
// this.folderId=folderId;
|
||||
manager=new WorkspaceFolderManager(folderId);
|
||||
manager.lock(processId);
|
||||
SynchFolderConfiguration folderConfig=manager.getSynchConfiguration();
|
||||
|
||||
try {
|
||||
descriptor=new ProcessDescriptor(folderId,
|
||||
manager.getTheFolder().get().getPath(),
|
||||
System.currentTimeMillis(),
|
||||
processId,
|
||||
operator,
|
||||
folderConfig);
|
||||
try {
|
||||
descriptor=new ProcessDescriptor(folderId, manager.getTheFolder().get().getPath(),System.currentTimeMillis(),processId,folderConfig);
|
||||
}catch(Exception e) {
|
||||
throw new WorkspaceInteractionException("Unable to read path from folder "+folderId,e);
|
||||
}
|
||||
|
|
@ -232,7 +225,7 @@ public class Process {
|
|||
}
|
||||
|
||||
String relativePath=toScanFolder.get().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
|
||||
ThreddsController folderController=new ThreddsController(relativePath);
|
||||
ThreddsController folderController=new ThreddsController(relativePath,config.getTargetToken());
|
||||
|
||||
RemoteFileDescriptor folderDesc=null;
|
||||
try{
|
||||
|
|
@ -312,13 +305,7 @@ public class Process {
|
|||
}catch(ItemNotFoundException e) {
|
||||
log.info("Creating folder {} under {} ",item,folderPath);
|
||||
folder=toScanFolder.newFolder(item, "Imported from thredds");
|
||||
WorkspaceUtils.initProperties(
|
||||
folder,relativePath+"/"+item ,
|
||||
config.getFilter(),
|
||||
ownerProcess.getDescriptor().getOperator().getContext(),
|
||||
config.getToCreateCatalogName(),
|
||||
config.getValidateMetadata(),
|
||||
config.getRootFolderId());
|
||||
WorkspaceUtils.initProperties(folder,relativePath+"/"+item , config.getFilter(), config.getTargetToken(),config.getToCreateCatalogName(),config.getValidateMetadata(),config.getRootFolderId());
|
||||
generateRequests(ownerProcess, service, folder);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl;
|
||||
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.User;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
|
|
@ -15,8 +14,6 @@ public class ProcessDescriptor implements Cloneable{
|
|||
private long launchTime;
|
||||
private String processId;
|
||||
|
||||
private User operator;
|
||||
|
||||
private SynchFolderConfiguration synchConfiguration;
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package org.gcube.usecases.ws.thredds.engine.impl;
|
|||
|
||||
import java.io.File;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
|
@ -18,8 +17,6 @@ import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
|
|||
import org.gcube.usecases.ws.thredds.Constants;
|
||||
import org.gcube.usecases.ws.thredds.LocalConfiguration;
|
||||
import org.gcube.usecases.ws.thredds.SyncEngine;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.User;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.ProcessInitializationThread;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.RequestLogger;
|
||||
import org.gcube.usecases.ws.thredds.faults.InternalException;
|
||||
|
|
@ -29,7 +26,6 @@ import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
|
|||
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
|
||||
import org.gcube.usecases.ws.thredds.model.CompletionCallback;
|
||||
import org.gcube.usecases.ws.thredds.model.ContainerType;
|
||||
import org.gcube.usecases.ws.thredds.model.SyncEngineStatusDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.model.SyncOperationCallBack;
|
||||
|
|
@ -123,13 +119,8 @@ public class SynchEngineImpl implements SyncEngine{
|
|||
|
||||
@Override
|
||||
public SyncFolderDescriptor check(String folderId, boolean recursively) throws WorkspaceInteractionException, InternalException {
|
||||
return new GuardedMethod<SyncFolderDescriptor>(folderId) {
|
||||
@Override
|
||||
protected SyncFolderDescriptor run() throws WorkspaceInteractionException, InternalException, Exception {
|
||||
WorkspaceFolderManager manager=getManager();
|
||||
return manager.check(recursively);
|
||||
}
|
||||
}.execute().getResult();
|
||||
WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId);
|
||||
return manager.check(recursively);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -140,73 +131,42 @@ public class SynchEngineImpl implements SyncEngine{
|
|||
|
||||
@Override
|
||||
public ProcessDescriptor doSync(String folderId) throws WorkspaceInteractionException, InternalException {
|
||||
return new GuardedMethod<ProcessDescriptor>(folderId) {
|
||||
@Override
|
||||
protected ProcessDescriptor run() throws ProcessNotFoundException, WorkspaceInteractionException, InternalException {
|
||||
User user=Security.getCurrent();
|
||||
log.info("User {} requested syncronization of {}",user,folderId);
|
||||
if(localProcesses.containsKey(folderId)) {
|
||||
ProcessDescriptor toReturn=localProcesses.get(folderId).getDescriptor();
|
||||
log.info("Returning ongoing process {} ",toReturn);
|
||||
return toReturn;
|
||||
|
||||
}
|
||||
else {
|
||||
WorkspaceFolderManager manager=getManager();
|
||||
if (!manager.isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not configured for synchronization.");
|
||||
if(manager.isLocked()) throw new WorkspaceLockedException("Folder "+folderId+"is locked by an external process.");
|
||||
if(!manager.isRoot()) throw new WorkspaceFolderNotRootException("Unable to launch synch operation. Folder "+folderId+" is not root configuration");
|
||||
|
||||
log.info("User {} is triggering synch for {} ",user,folderId);
|
||||
|
||||
Process toLaunch=new Process(user,folderId,completionCallback);
|
||||
localProcesses.put(folderId, toLaunch);
|
||||
initializationExecutor.submit(new ProcessInitializationThread(toLaunch,synchronizationExecutor));
|
||||
return toLaunch.getDescriptor();
|
||||
}
|
||||
}
|
||||
}.execute().getResult();
|
||||
if(localProcesses.containsKey(folderId))
|
||||
return localProcesses.get(folderId).getDescriptor();
|
||||
else {
|
||||
WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId);
|
||||
if (!manager.isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not configured for synchronization.");
|
||||
if(manager.isLocked()) throw new WorkspaceLockedException("Folder "+folderId+"is locked by an external process.");
|
||||
if(!manager.isRoot()) throw new WorkspaceFolderNotRootException("Unable to launch synch operation. Folder "+folderId+" is not root configuration");
|
||||
Process toLaunch=new Process(folderId,completionCallback);
|
||||
localProcesses.put(folderId, toLaunch);
|
||||
initializationExecutor.submit(new ProcessInitializationThread(toLaunch,synchronizationExecutor));
|
||||
return toLaunch.getDescriptor();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopSynch(String folderId) throws ProcessNotFoundException, WorkspaceInteractionException, InternalException {
|
||||
new GuardedMethod(folderId) {
|
||||
@Override
|
||||
protected Object run() throws ProcessNotFoundException {
|
||||
if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under local processes");
|
||||
localProcesses.get(folderId).cancel();
|
||||
return null;
|
||||
}
|
||||
}.execute();
|
||||
|
||||
public void stopSynch(String folderId) throws ProcessNotFoundException {
|
||||
if(!localProcesses.containsKey(folderId)) throw new ProcessNotFoundException(folderId+" is not under local processes");
|
||||
localProcesses.get(folderId).cancel();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void setSynchronizedFolder(SynchFolderConfiguration config,String folderId) throws WorkspaceInteractionException, InternalException {
|
||||
|
||||
// Check config
|
||||
if(config==null) throw new InternalException("Passed config is null : "+config);
|
||||
String remotePath=config.getRemotePath();
|
||||
if(remotePath==null||remotePath.isEmpty()||remotePath.startsWith("/"))
|
||||
throw new InternalException("Invalid remote path "+remotePath+".");
|
||||
User operator=Security.getCurrent();
|
||||
if(!operator.getContext().equals(config.getTargetContext()))
|
||||
throw new SecurityException("Invalid operation. Configuration pointed to "+config.getTargetContext()+" while operator is in "+operator.getContext());
|
||||
|
||||
|
||||
|
||||
new WorkspaceFolderManager(folderId).configure(config);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unsetSynchronizedFolder(String folderId,boolean deleteRemoteContent) throws WorkspaceInteractionException, InternalException {
|
||||
new GuardedMethod(folderId) {
|
||||
@Override
|
||||
protected Object run() throws WorkspaceInteractionException, InternalException {
|
||||
getManager().dismiss(deleteRemoteContent);
|
||||
return null;
|
||||
}
|
||||
}.execute();
|
||||
new WorkspaceFolderManager(folderId).dismiss(deleteRemoteContent);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -215,29 +175,20 @@ public class SynchEngineImpl implements SyncEngine{
|
|||
}
|
||||
|
||||
@Override
|
||||
public void updateCatalogFile(String folderId, File toUpdate) throws InternalException, WorkspaceInteractionException {
|
||||
|
||||
new GuardedMethod(folderId) {
|
||||
@Override
|
||||
protected Object run() throws Exception {
|
||||
try {
|
||||
log.warn("Forcing unlock of {} ",folderId);
|
||||
WorkspaceFolderManager manager=getManager();
|
||||
File previousCatalogFile=manager.loadCatalogFile();
|
||||
String lockId=UUID.randomUUID().toString();
|
||||
manager.lock(lockId);
|
||||
manager.updateCatalogFile(toUpdate);
|
||||
manager.unlock(lockId);
|
||||
return null;
|
||||
}catch(SecurityException e) {
|
||||
throw e;
|
||||
}catch(Throwable t) {
|
||||
log.warn("Unable to update catalogFile for {}. Trying to restore previous one..",folderId,t);
|
||||
throw new InternalException("Unable to restore previous catalog.",t);
|
||||
//TODO try to restore previous catalog
|
||||
}
|
||||
}
|
||||
}.execute();
|
||||
public void updateCatalogFile(String folderId, File toUpdate) throws InternalException {
|
||||
File previousCatalogFile=null;
|
||||
try {
|
||||
WorkspaceFolderManager manager=new WorkspaceFolderManager(folderId);
|
||||
previousCatalogFile=manager.loadCatalogFile();
|
||||
String lockId=UUID.randomUUID().toString();
|
||||
manager.lock(lockId);
|
||||
manager.updateCatalogFile(toUpdate);
|
||||
manager.unlock(lockId);
|
||||
}catch(Throwable t) {
|
||||
log.warn("Unable to update catalogFile for {}. Trying to restore previous one..",folderId,t);
|
||||
throw new InternalException("Unable to restore previous catalog.",t);
|
||||
//TODO try to restore previous catalog
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -262,14 +213,8 @@ public class SynchEngineImpl implements SyncEngine{
|
|||
|
||||
@Override
|
||||
public void forceUnlock(String folderId) throws InternalException, WorkspaceInteractionException {
|
||||
new GuardedMethod(folderId) {
|
||||
@Override
|
||||
protected Object run() throws WorkspaceInteractionException, InternalException {
|
||||
log.warn("Forcing unlock of {} ",folderId);
|
||||
getManager().forceUnlock();
|
||||
return null;
|
||||
}
|
||||
}.execute();
|
||||
log.warn("Forcing unlock of {} ",folderId);
|
||||
new WorkspaceFolderManager(folderId).forceUnlock();
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -287,8 +232,8 @@ public class SynchEngineImpl implements SyncEngine{
|
|||
|
||||
|
||||
@Override
|
||||
public Set<CatalogBean> getAvailableCatalogs() throws InternalException {
|
||||
ThreddsController controller=new ThreddsController("");
|
||||
public Set<CatalogBean> getAvailableCatalogsByToken(String token) throws InternalException {
|
||||
ThreddsController controller=new ThreddsController("",token);
|
||||
ThreddsInfo info=controller.getThreddsInfo();
|
||||
Set<CatalogBean> toReturn=asCatalogBeanSet(info.getCatalog());
|
||||
DataSetScan mainScan=info.getCatalog().getDeclaredDataSetScan().iterator().next();
|
||||
|
|
@ -330,12 +275,4 @@ public class SynchEngineImpl implements SyncEngine{
|
|||
ThreadPoolExecutor exec=(ThreadPoolExecutor) synchronizationExecutor;
|
||||
return new SyncEngineStatusDescriptor(exec.getActiveCount(), exec.getQueue().size(), LocalConfiguration.get().asMap());
|
||||
}
|
||||
|
||||
@Override
|
||||
public SynchFolderConfiguration getConfig(String folderId) throws WorkspaceInteractionException, WorkspaceNotSynchedException {
|
||||
return new WorkspaceFolderManager(folderId).getSynchConfiguration();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
|||
import org.gcube.common.resources.gcore.GCoreEndpoint;
|
||||
import org.gcube.data.transfer.library.DataTransferClient;
|
||||
import org.gcube.data.transfer.library.TransferResult;
|
||||
import org.gcube.data.transfer.library.client.AuthorizationFilter;
|
||||
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
|
|
@ -43,6 +42,7 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|||
import org.gcube.spatial.data.sdi.model.ServiceConstants;
|
||||
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
|
||||
import org.gcube.usecases.ws.thredds.Constants;
|
||||
import org.gcube.usecases.ws.thredds.TokenSetter;
|
||||
import org.gcube.usecases.ws.thredds.faults.InternalException;
|
||||
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
|
||||
import org.gcube.usecases.ws.thredds.faults.UnableToLockException;
|
||||
|
|
@ -66,53 +66,62 @@ public class ThreddsController {
|
|||
|
||||
|
||||
|
||||
private URL threddsEndpoints;
|
||||
private String hostname;
|
||||
private String operatingPath;
|
||||
private String targetToken;
|
||||
|
||||
public ThreddsController(String path) throws InternalException {
|
||||
public ThreddsController(String path,String targetToken) throws InternalException {
|
||||
operatingPath=path;
|
||||
this.targetToken=targetToken;
|
||||
|
||||
threddsEndpoints=getThreddsEndpoint();
|
||||
if(threddsEndpoints==null) throw new InternalException("Invalid hostnam in context "+ScopeUtils.getCurrentScope());
|
||||
hostname=getThreddsHost();
|
||||
if(hostname==null) throw new InternalException("Invalid hostnam in context "+ScopeUtils.getCurrentScope());
|
||||
}
|
||||
|
||||
// private static final String truncate(String toTruncate) {
|
||||
// return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"...";
|
||||
// }
|
||||
private static final String truncate(String toTruncate) {
|
||||
return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"...";
|
||||
}
|
||||
|
||||
private String callerToken=null;
|
||||
|
||||
// private void setTargetToken() {
|
||||
// if(callerToken==null) {
|
||||
// callerToken=TokenSetter.getCurrentToken();
|
||||
// log.trace("Storing caller token {}. Target Token is {}",truncate(callerToken),truncate(targetToken));
|
||||
// TokenSetter.setToken(targetToken);
|
||||
// }else {
|
||||
// log.trace("Caller token {} already registered. Target Token is {}",truncate(callerToken),truncate(targetToken));
|
||||
// }
|
||||
// }
|
||||
private void setTargetToken() {
|
||||
if(callerToken==null) {
|
||||
callerToken=TokenSetter.getCurrentToken();
|
||||
log.trace("Storing caller token {}. Target Token is {}",truncate(callerToken),truncate(targetToken));
|
||||
TokenSetter.setToken(targetToken);
|
||||
}else {
|
||||
log.trace("Caller token {} already registered. Target Token is {}",truncate(callerToken),truncate(targetToken));
|
||||
}
|
||||
}
|
||||
|
||||
// private void resetCallerToken() {
|
||||
// if(callerToken!=null) {
|
||||
// log.trace(String.format("Resetting caller token %1$s. Target Token is %2$s, current is %3$s ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())));
|
||||
// TokenSetter.setToken(callerToken);
|
||||
// callerToken=null;
|
||||
// }else log.trace(String.format("Caller token %1$s already reset [current token %2$s]. Target Token is %3$s",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)));
|
||||
// }
|
||||
private void resetCallerToken() {
|
||||
if(callerToken!=null) {
|
||||
log.trace(String.format("Resetting caller token %1$s. Target Token is %2$s, current is %3$s ",truncate(callerToken),truncate(targetToken),truncate(TokenSetter.getCurrentToken())));
|
||||
TokenSetter.setToken(callerToken);
|
||||
callerToken=null;
|
||||
}else log.trace(String.format("Caller token %1$s already reset [current token %2$s]. Target Token is %3$s",truncate(callerToken),truncate(TokenSetter.getCurrentToken()),truncate(targetToken)));
|
||||
}
|
||||
|
||||
public final ThreddsInfo getThreddsInfo() {
|
||||
String infoPath="https://"+threddsEndpoints.getHost()+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";;
|
||||
setTargetToken();
|
||||
try{
|
||||
String infoPath="https://"+hostname+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";;
|
||||
log.info("Loading thredds info from {} ",infoPath);
|
||||
WebTarget target=getWebClient().target(infoPath);
|
||||
return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class);
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void lockFolder(String processId) throws UnableToLockException {
|
||||
setTargetToken();
|
||||
PrintWriter writer=null;
|
||||
File temp=null;
|
||||
try{
|
||||
log.info("Locking remote path {} to processId {} ",operatingPath,processId);
|
||||
DataTransferClient cl=getDTClient(threddsEndpoints);
|
||||
DataTransferClient cl=getDTClient(hostname);
|
||||
|
||||
Destination dest=new Destination();
|
||||
dest.setCreateSubfolders(true);
|
||||
|
|
@ -138,17 +147,21 @@ public class ThreddsController {
|
|||
}catch(IOException e) {
|
||||
log.warn("Unable to delete temp file {} ",temp.getAbsolutePath(),e);
|
||||
}
|
||||
resetCallerToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void deleteThreddsFile(String location) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
setTargetToken();
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
log.info("Deleting file at {} ",urlString);
|
||||
try{
|
||||
getWebClient().target(urlString).request().delete();
|
||||
}catch(Throwable t) {
|
||||
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
|
||||
}finally{
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -163,21 +176,25 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
public String readThreddsFile(String location) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
setTargetToken();
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
log.info("Reading file at {} ",urlString);
|
||||
try{
|
||||
return getWebClient().target(urlString).request().get().readEntity(String.class);
|
||||
}catch(Throwable t) {
|
||||
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void createEmptyFolder(String targetPath) throws InternalException {
|
||||
setTargetToken();
|
||||
String toCleanPath=getPathFromStartingLocation(targetPath);
|
||||
try{
|
||||
log.info("Cleaning up {} on {} ",toCleanPath,threddsEndpoints);
|
||||
DataTransferClient client=getDTClient(threddsEndpoints);
|
||||
log.info("Cleaning up {} on {} ",toCleanPath,hostname);
|
||||
DataTransferClient client=getDTClient(hostname);
|
||||
File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||
toTransfer.createNewFile();
|
||||
Destination dest=new Destination();
|
||||
|
|
@ -188,7 +205,7 @@ public class ThreddsController {
|
|||
dest.setSubFolder(toCleanPath);
|
||||
dest.setDestinationFileName(toTransfer.getName());
|
||||
|
||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),threddsEndpoints);
|
||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
||||
client.localFile(toTransfer, dest);
|
||||
this.deleteThreddsFile(targetPath+"/"+toTransfer.getName());
|
||||
log.info("Done");
|
||||
|
|
@ -196,10 +213,13 @@ public class ThreddsController {
|
|||
}catch(Exception e) {
|
||||
log.error("Unable to delete remote folder "+toCleanPath,e);
|
||||
throw new InternalException("Unable to cleanup remote folder.");
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
public ThreddsCatalog createCatalog(String name) throws InternalException {
|
||||
setTargetToken();
|
||||
try{
|
||||
SecurityTokenProvider.instance.get();
|
||||
log.info("Creating catalog with name {} for path {} ",name,operatingPath);
|
||||
|
|
@ -214,13 +234,19 @@ public class ThreddsController {
|
|||
}catch(Throwable t) {
|
||||
log.error("Unable to create catalog",t);
|
||||
throw new InternalException("Unable to create catalog",t);
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
public ThreddsCatalog getCatalog() {
|
||||
ThreddsInfo info=getThreddsInfo();
|
||||
setTargetToken();
|
||||
try{ThreddsInfo info=getThreddsInfo();
|
||||
String instanceBasePath=info.getLocalBasePath();
|
||||
return info.getCatalogByFittingLocation(instanceBasePath+"/"+operatingPath);
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -229,32 +255,41 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
public RemoteFileDescriptor getFileDescriptor(String path) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
setTargetToken();
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
log.info("Reading file at {} ",urlString);
|
||||
try{
|
||||
return getWebClient().target(urlString).queryParam("descriptor", true).request().get().readEntity(RemoteFileDescriptor.class);
|
||||
}catch(Throwable t) {
|
||||
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
public InputStream getInputStream(String path) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
setTargetToken();
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
log.info("Reading file at {} ",urlString);
|
||||
try{
|
||||
return getWebClient().target(urlString).request().get().readEntity(InputStream.class);
|
||||
}catch(Throwable t) {
|
||||
throw new RemoteFileNotFoundException("Unable to access "+urlString, t);
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public TransferResult transferFile(Destination dest,URL url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
|
||||
DataTransferClient client=getDTClient(threddsEndpoints);
|
||||
setTargetToken();
|
||||
try{DataTransferClient client=getDTClient(hostname);
|
||||
if(invocations!=null&&!invocations.isEmpty())
|
||||
return client.httpSource(url, dest,invocations);
|
||||
else return client.httpSource(url, dest);
|
||||
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -268,8 +303,8 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
|
||||
private URL getThreddsEndpoint() throws InternalException{
|
||||
|
||||
private String getThreddsHost() throws InternalException{
|
||||
setTargetToken();
|
||||
try{
|
||||
String sdiUrl="https://"+getSDIServiceHost()+"/"+Constants.SDI_CONFIG_PATH;
|
||||
log.info("checking sdI configuration at {}",sdiUrl);
|
||||
|
|
@ -279,10 +314,12 @@ public class ThreddsController {
|
|||
throw new InternalException("Failed to contact SDI. Message "+respString);
|
||||
|
||||
DocumentContext sourceCtx=JsonPath.using(JSON_PATH_ALWAYS_LIST_CONFIG).parse(respString);
|
||||
return new URL(((List<String>) sourceCtx.read("$.threddsConfiguration..baseEndpoint")).get(0));
|
||||
return new URL(((List<String>) sourceCtx.read("$.threddsConfiguration..baseEndpoint")).get(0)).getHost();
|
||||
}catch(Throwable t) {
|
||||
log.error("Unable to read SDI configuration",t);
|
||||
throw new InternalException("Unable to gt configuration from SDI",t);
|
||||
}finally {
|
||||
resetCallerToken();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -300,12 +337,12 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
private static Client getWebClient() {
|
||||
return ClientBuilder.newClient(new ClientConfig().register(AuthorizationFilter.class))
|
||||
return ClientBuilder.newClient(new ClientConfig().register(CustomAuthorizationFilter.class))
|
||||
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
|
||||
}
|
||||
|
||||
private static DataTransferClient getDTClient(URL threddsEndpoint) throws UnreachableNodeException, ServiceNotFoundException {
|
||||
log.debug("Getting DT Client for {} ",threddsEndpoint);
|
||||
return DataTransferClient.getInstanceByEndpoint(threddsEndpoint.toString());
|
||||
private static DataTransferClient getDTClient(String threddsHostName) throws UnreachableNodeException, ServiceNotFoundException {
|
||||
log.debug("Getting DT Client for {} ",threddsHostName);
|
||||
return DataTransferClient.getInstanceByEndpoint("https://"+threddsHostName);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,13 +51,9 @@ public class WorkspaceFolderManager {
|
|||
|
||||
private StorageHubClient sc;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public WorkspaceFolderManager(String folderId) throws WorkspaceInteractionException {
|
||||
try{
|
||||
|
||||
// ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
|
||||
sc=WorkspaceUtils.getClient();
|
||||
theFolder=sc.open(folderId).asFolder();
|
||||
this.folderId=folderId;
|
||||
|
|
@ -74,7 +70,7 @@ public class WorkspaceFolderManager {
|
|||
public ThreddsController getThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
|
||||
if(threddsController==null) {
|
||||
SynchFolderConfiguration config=getSynchConfiguration();
|
||||
threddsController=new ThreddsController(config.getRemotePath());
|
||||
threddsController=new ThreddsController(config.getRemotePath(),config.getTargetToken());
|
||||
}
|
||||
return threddsController;
|
||||
}
|
||||
|
|
@ -82,8 +78,8 @@ public class WorkspaceFolderManager {
|
|||
private ThreddsController getRootThreddsController() throws WorkspaceNotSynchedException, WorkspaceInteractionException, InternalException {
|
||||
try {
|
||||
FolderContainer root=sc.open(getSynchConfiguration().getRootFolderId()).asFolder();
|
||||
SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root).getConfiguration();
|
||||
return new ThreddsController(rootConfig.getRemotePath());
|
||||
SynchFolderConfiguration rootConfig=WorkspaceUtils.loadConfiguration(root);
|
||||
return new ThreddsController(rootConfig.getRemotePath(),rootConfig.getTargetToken());
|
||||
}catch(StorageHubException e) {
|
||||
throw new WorkspaceInteractionException(e);
|
||||
}
|
||||
|
|
@ -100,7 +96,7 @@ public class WorkspaceFolderManager {
|
|||
try {
|
||||
if(!isSynched()) throw new WorkspaceNotSynchedException("Folder "+folderId+" is not synched.");
|
||||
log.debug("Loading properties for ");
|
||||
config=WorkspaceUtils.loadConfiguration(theFolder).getConfiguration();
|
||||
config=WorkspaceUtils.loadConfiguration(theFolder);
|
||||
}catch(StorageHubException e) {
|
||||
throw new WorkspaceInteractionException("Unable to load synch configuration in "+folderId,e);
|
||||
}
|
||||
|
|
@ -121,15 +117,7 @@ public class WorkspaceFolderManager {
|
|||
SynchFolderConfiguration config=getSynchConfiguration();
|
||||
try{
|
||||
checkFolder(theFolder,recursively,config,null,theFolder.getId(),WorkspaceUtils.safelyGetLastUpdate(theFolder.get()));
|
||||
SynchronizationStatus resultingStatus=
|
||||
SynchronizationStatus.valueOf(theFolder.get().getMetadata().getMap().get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)+"");
|
||||
|
||||
log.info("Resulting status for {} IS {} ",folderId,resultingStatus);
|
||||
|
||||
return new SyncFolderDescriptor(resultingStatus,
|
||||
this.folderId,
|
||||
this.theFolder.get().getPath(),
|
||||
config,isLocked());
|
||||
return new SyncFolderDescriptor(this.folderId,this.theFolder.get().getPath(),config);
|
||||
}catch(StorageHubException e) {
|
||||
throw new WorkspaceInteractionException(e);
|
||||
}
|
||||
|
|
@ -155,7 +143,7 @@ public class WorkspaceFolderManager {
|
|||
if(isSynched()) throw new WorkspaceInteractionException("Folder "+folderId+" is already configured for synchronization.");
|
||||
log.info("Configuring folder {} as {} ",folderId,toSet);
|
||||
|
||||
|
||||
|
||||
|
||||
// Checking AND initializing remote folder
|
||||
log.debug("Checking remote folder existence .. ");
|
||||
|
|
@ -163,7 +151,7 @@ public class WorkspaceFolderManager {
|
|||
try {
|
||||
String catalogName=toSet.getToCreateCatalogName();
|
||||
|
||||
ThreddsController controller= new ThreddsController(toSet.getRemotePath());
|
||||
ThreddsController controller= new ThreddsController(toSet.getRemotePath(),toSet.getTargetToken());
|
||||
if(!controller.existsThreddsFile(null)) {
|
||||
log.info("Folder not found, creating it..");
|
||||
controller.createEmptyFolder(null);
|
||||
|
|
@ -186,7 +174,7 @@ public class WorkspaceFolderManager {
|
|||
}
|
||||
|
||||
WorkspaceUtils.initProperties(theFolder, toSet.getRemotePath(), toSet.getFilter(),
|
||||
toSet.getTargetContext(),toSet.getToCreateCatalogName(),toSet.getValidateMetadata(),theFolder.getId());
|
||||
toSet.getTargetToken(),toSet.getToCreateCatalogName(),toSet.getValidateMetadata(),theFolder.getId());
|
||||
|
||||
}catch(InternalException e) {
|
||||
throw new InternalException ("Unable to check/initialize remote folder",e);
|
||||
|
|
@ -253,12 +241,7 @@ public class WorkspaceFolderManager {
|
|||
|
||||
|
||||
|
||||
private static void checkFolder(FolderContainer folder,
|
||||
boolean recursive,
|
||||
SynchFolderConfiguration rootConfig,
|
||||
String relativePathFromRootFolder,
|
||||
String rootFolderId,
|
||||
Date lastUpdatedRoutine) throws StorageHubException, InternalException {
|
||||
private static void checkFolder(FolderContainer folder,boolean recursive, SynchFolderConfiguration rootConfig, String relativePathFromRootFolder, String rootFolderId,Date lastUpdatedRoutine) throws StorageHubException, InternalException {
|
||||
// Check folder configuration
|
||||
log.trace("Checking folder {} ",folder.get().getPath());
|
||||
log.debug("Configuration is {}, relativePath is {} ",rootConfig,relativePathFromRootFolder);
|
||||
|
|
@ -269,7 +252,7 @@ public class WorkspaceFolderManager {
|
|||
|
||||
|
||||
|
||||
ThreddsController controller=new ThreddsController(currentRemotePath);
|
||||
ThreddsController controller=new ThreddsController(currentRemotePath, rootConfig.getTargetToken());
|
||||
|
||||
|
||||
HashSet<String> currentFolderExistingItem=new HashSet<String>();
|
||||
|
|
@ -278,7 +261,7 @@ public class WorkspaceFolderManager {
|
|||
log.debug("Initializing properties for {} ",folderName);
|
||||
//INIT PROPERTIES IF NOT PRESENT
|
||||
if(!WorkspaceUtils.isConfigured(folder.get()))
|
||||
WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetContext(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
|
||||
WorkspaceUtils.initProperties(folder,currentRemotePath,rootConfig.getFilter(),rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
|
||||
|
||||
for(ItemContainer<?> item:folder.list().withAccounting().withMetadata().getContainers()) {
|
||||
String itemName=item.get().getName();
|
||||
|
|
@ -287,9 +270,7 @@ public class WorkspaceFolderManager {
|
|||
if(item.getType().equals(ContainerType.FOLDER)) {
|
||||
if(recursive)
|
||||
checkFolder((FolderContainer) item,recursive,rootConfig,itemRelativePath,rootFolderId,lastUpdatedRoutine);
|
||||
else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(),
|
||||
rootConfig.getTargetContext(),
|
||||
rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
|
||||
else WorkspaceUtils.initProperties(item, itemRemotePath, rootConfig.getFilter(), rootConfig.getTargetToken(),rootConfig.getToCreateCatalogName(),rootConfig.getValidateMetadata(),rootFolderId);
|
||||
}else if(rootConfig.matchesFilter(itemName)) {
|
||||
if(!WorkspaceUtils.isConfigured(item.get()))
|
||||
WorkspaceUtils.initProperties(item, null, null, null,null,null,null);
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl;
|
||||
|
||||
import java.text.ParsePosition;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
|
||||
|
||||
import org.gcube.common.storagehub.client.dsl.ContainerType;
|
||||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||||
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
||||
|
|
@ -21,20 +21,14 @@ import org.gcube.common.storagehub.model.items.nodes.accounting.AccountFolderEnt
|
|||
import org.gcube.common.storagehub.model.types.WorkspaceItemType;
|
||||
import org.gcube.data.transfer.model.RemoteFileDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.Constants;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.DeleteRemoteRequest;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.SynchronizationThread;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.threads.TransferFromThreddsRequest;
|
||||
import org.gcube.usecases.ws.thredds.faults.InternalException;
|
||||
import org.gcube.usecases.ws.thredds.faults.ItemNotFoundException;
|
||||
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
|
||||
import org.gcube.usecases.ws.thredds.model.StepReport;
|
||||
import org.gcube.usecases.ws.thredds.model.StepReport.Status;
|
||||
import org.gcube.usecases.ws.thredds.model.SyncFolderDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
|
@ -225,7 +219,7 @@ public class WorkspaceUtils {
|
|||
}
|
||||
|
||||
|
||||
static void initProperties(ItemContainer<?> toInit, String remotePath, String filter, String targetContext,
|
||||
static void initProperties(ItemContainer<?> toInit, String remotePath, String filter, String targetToken,
|
||||
String catalogName,Boolean validateMeta, String rootFolderId) throws StorageHubException {
|
||||
|
||||
|
||||
|
|
@ -242,12 +236,7 @@ public class WorkspaceUtils {
|
|||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.SYNCH_FILTER,filter);
|
||||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PATH,remotePath);
|
||||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.REMOTE_PERSISTENCE,Constants.THREDDS_PERSISTENCE);
|
||||
|
||||
|
||||
//NB OLDR VERSION OF CONFIGS MAY CONTAIN TOKEN INSTEAD OF CONTEXT
|
||||
// initIfMissing(toSetProperties,Constants.WorkspaceProperties.TARGET_TOKEN,targetToken);
|
||||
|
||||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.TARGET_CONTEXT,targetContext);
|
||||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.TARGET_TOKEN,targetToken);
|
||||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.RELATED_CATALOG,catalogName);
|
||||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.VALIDATE_METADATA,validateMeta+"");
|
||||
initIfMissing(toSetProperties,Constants.WorkspaceProperties.ROOT_FOLDER_ID,rootFolderId);
|
||||
|
|
@ -366,56 +355,18 @@ public class WorkspaceUtils {
|
|||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* Side effect : upgrades node configuration
|
||||
* If called on a leaf ITEM, the configuration is taken from the parent node
|
||||
*
|
||||
* NB doesn't interact with Thredds, thus it lacks locked information
|
||||
*
|
||||
* @param map
|
||||
* @param type
|
||||
* @return
|
||||
* @throws WorkspaceNotSynchedException
|
||||
*/
|
||||
public static SyncFolderDescriptor loadConfiguration(ItemContainer<?> item) throws StorageHubException, WorkspaceInteractionException {
|
||||
public static SynchFolderConfiguration loadConfiguration(ItemContainer<?> item) throws StorageHubException {
|
||||
if(item.getType().equals(ContainerType.FOLDER)) {
|
||||
Metadata meta=item.get().getMetadata();
|
||||
Map<String,Object> map=meta.getMap();
|
||||
|
||||
SyncFolderDescriptor desc = (SyncFolderDescriptor) getInfo(map, item.getType());
|
||||
desc.setFolderId(item.getId());
|
||||
desc.setFolderPath(item.get().getPath());
|
||||
|
||||
|
||||
|
||||
String ctx=desc.getConfiguration().getTargetContext();
|
||||
if(ctx==null||ctx.isEmpty()||ctx.equalsIgnoreCase("null")) {
|
||||
|
||||
String itemId=item.getId();
|
||||
log.warn("CTX IS NULL, Trying to convert legacy configuration for item {} ",itemId);
|
||||
|
||||
try {
|
||||
Object tkn=map.get(Constants.WorkspaceProperties.TARGET_TOKEN);
|
||||
String context=Security.getContextFromgcubeToken(tkn+"");
|
||||
log.info("Obtained context {} for folderId {} ",context,itemId);
|
||||
map.put(Constants.WorkspaceProperties.TARGET_TOKEN, null);
|
||||
map.put(Constants.WorkspaceProperties.TARGET_CONTEXT, context);
|
||||
meta.setMap(map);
|
||||
item.setMetadata(meta);
|
||||
log.info("Converted legacy configuration for item {}. Context is ",itemId,context);
|
||||
return loadConfiguration(item);
|
||||
}catch(Exception e) {
|
||||
throw new WorkspaceInteractionException("Invalid configuration in folder "+itemId+", cannot convert.");
|
||||
}
|
||||
}
|
||||
|
||||
return desc;
|
||||
Map<String,Object> map=item.get().getMetadata().getMap();
|
||||
SynchFolderConfiguration config=new SynchFolderConfiguration();
|
||||
config.setFilter(""+map.get(Constants.WorkspaceProperties.SYNCH_FILTER));
|
||||
config.setRemotePath(""+map.get(Constants.WorkspaceProperties.REMOTE_PATH));
|
||||
config.setRemotePersistence(""+map.get(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
|
||||
config.setTargetToken(""+map.get(Constants.WorkspaceProperties.TARGET_TOKEN));
|
||||
config.setToCreateCatalogName(""+map.get(Constants.WorkspaceProperties.RELATED_CATALOG));
|
||||
config.setValidateMetadata(Boolean.parseBoolean(""+map.get(Constants.WorkspaceProperties.VALIDATE_METADATA)));
|
||||
config.setRootFolderId(""+map.get(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
|
||||
return config;
|
||||
}else {
|
||||
FolderContainer parentFolder=getClient().open(item.get().getParentId()).asFolder();
|
||||
|
||||
|
|
@ -467,59 +418,5 @@ public class WorkspaceUtils {
|
|||
throw new ItemNotFoundException("Unable to find "+path);
|
||||
}
|
||||
|
||||
//************** parsing properties
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* Does not send request, thus
|
||||
* for items it only has basic generic info
|
||||
* for Folders the bean lacks folderId, folder path, locked
|
||||
*
|
||||
* @param map
|
||||
* @param type
|
||||
* @return
|
||||
* @throws WorkspaceNotSynchedException
|
||||
*/
|
||||
public static SynchronizedElementInfo getInfo(Map<String,Object> map,ContainerType type) throws WorkspaceNotSynchedException {
|
||||
|
||||
if(!(map.containsKey(Constants.WorkspaceProperties.TBS)&&Boolean.parseBoolean(map.get(Constants.WorkspaceProperties.TBS)+"")))
|
||||
throw new WorkspaceNotSynchedException("Item is not to be synchronized");
|
||||
|
||||
SynchronizationStatus status = SynchronizationStatus.valueOf(map.get(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS)+"");
|
||||
SynchronizedElementInfo toReturn=null;
|
||||
if(type.equals(ContainerType.FOLDER)) {
|
||||
|
||||
SynchFolderConfiguration config=new SynchFolderConfiguration();
|
||||
config.setFilter(""+map.get(Constants.WorkspaceProperties.SYNCH_FILTER));
|
||||
config.setRemotePath(""+map.get(Constants.WorkspaceProperties.REMOTE_PATH));
|
||||
config.setRemotePersistence(""+map.get(Constants.WorkspaceProperties.REMOTE_PERSISTENCE));
|
||||
|
||||
Object ctx=map.get(Constants.WorkspaceProperties.TARGET_CONTEXT);
|
||||
config.setTargetContext(ctx+"");
|
||||
|
||||
|
||||
|
||||
config.setToCreateCatalogName(""+map.get(Constants.WorkspaceProperties.RELATED_CATALOG));
|
||||
config.setValidateMetadata(Boolean.parseBoolean(""+map.get(Constants.WorkspaceProperties.VALIDATE_METADATA)));
|
||||
config.setRootFolderId(""+map.get(Constants.WorkspaceProperties.ROOT_FOLDER_ID));
|
||||
|
||||
toReturn=new SyncFolderDescriptor(status,
|
||||
"",//folder id
|
||||
"",// folder path
|
||||
config,null);
|
||||
|
||||
}if(type.equals(ContainerType.FILE)) {
|
||||
toReturn=new SynchronizedElementInfo(status);
|
||||
}
|
||||
//COMMON FIELDS
|
||||
toReturn.setLastupdateTime(map.get(Constants.WorkspaceProperties.LAST_UPDATE_TIME)+"");
|
||||
toReturn.setLastSynchronizationStatus(Status.valueOf(map.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+""));
|
||||
|
||||
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl.security;
|
||||
|
||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
|
||||
import org.gcube.common.authorization.library.provider.ClientInfo;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.authorization.library.provider.AccessTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class Security {
|
||||
|
||||
|
||||
|
||||
public static User getCurrent() throws SecurityException {
|
||||
String context=ScopeProvider.instance.get();
|
||||
if(context==null) throw new SecurityException("Cannot determine context");
|
||||
|
||||
log.debug("Context is {}, checking tokens..",context);
|
||||
|
||||
ClientInfo client = null;
|
||||
try{
|
||||
AuthorizationProvider.instance.get().getClient();
|
||||
}catch(Throwable e) {
|
||||
log.warn("Unable to get client info ",e);
|
||||
}
|
||||
User toReturn = new User(client,AccessTokenProvider.instance.get(),SecurityTokenProvider.instance.get(),context);
|
||||
|
||||
log.info("Current User is {} ",toReturn);
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
public static void set(User toSet) {
|
||||
//cleanup everything
|
||||
resetUser();
|
||||
log.debug("Setting User {} ",toSet);
|
||||
if(toSet.getUma_token()!=null)AccessTokenProvider.instance.set(toSet.getUma_token());
|
||||
if(toSet.getGcube_token()!=null)SecurityTokenProvider.instance.set(toSet.getGcube_token());
|
||||
if(toSet.getContext()!=null)ScopeProvider.instance.set(toSet.getContext());
|
||||
}
|
||||
|
||||
|
||||
public static void resetUser(){
|
||||
log.debug("Resetting user");
|
||||
SecurityTokenProvider.instance.reset();
|
||||
AccessTokenProvider.instance.reset();
|
||||
ScopeProvider.instance.reset();
|
||||
}
|
||||
public static void checkOperator(SynchFolderConfiguration config) throws SecurityException{
|
||||
User current=getCurrent();
|
||||
log.debug("Checking if current user {} can synch {} ",getCurrent(), config);
|
||||
|
||||
// check same vre
|
||||
String expectedContext=config.getTargetContext();
|
||||
String currentContext=current.getContext();
|
||||
|
||||
if(!expectedContext.equals(currentContext))
|
||||
throw new SecurityException("Illegal access to folder [root : "+config.getRootFolderId()+", expected context : "+expectedContext+"] from context "+currentContext);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static String getContextFromgcubeToken(String token) throws ObjectNotFound, Exception {
|
||||
log.debug("Checking context of gcube-token {}...",token.substring(0,6));
|
||||
User caller=getCurrent();
|
||||
try {
|
||||
resetUser();
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
return entry.getContext();
|
||||
}finally {
|
||||
log.debug("Resetting user "+caller);
|
||||
set(caller);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl.security;
|
||||
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceInteractionException;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Setter
|
||||
@Getter
|
||||
public class SecurityException extends WorkspaceInteractionException {
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = -485206587548325970L;
|
||||
private User caller;
|
||||
private SynchFolderConfiguration folderConfiguration;
|
||||
|
||||
public SecurityException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SecurityException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SecurityException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SecurityException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public SecurityException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl.security;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.ClientInfo;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Getter
|
||||
public class User{
|
||||
private ClientInfo user;
|
||||
private String uma_token;
|
||||
|
||||
private String gcube_token;
|
||||
|
||||
private String context;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("User [user=");
|
||||
builder.append(user);
|
||||
builder.append(", uma_token=");
|
||||
builder.append(uma_token==null?uma_token:"***");
|
||||
|
||||
builder.append(", gcube_token=");
|
||||
builder.append(gcube_token==null?gcube_token:"***");
|
||||
|
||||
builder.append(", context=");
|
||||
builder.append(context);
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -36,8 +36,6 @@ import org.gcube.usecases.ws.thredds.NetUtils;
|
|||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.User;
|
||||
import org.gcube.usecases.ws.thredds.faults.CancellationException;
|
||||
import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError;
|
||||
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
|
||||
|
|
@ -71,15 +69,11 @@ public class SynchronizationThread implements Runnable {
|
|||
|
||||
FolderItem parentFolderItem=theRequest.getLocation();
|
||||
try {
|
||||
User operator=theRequest.getProcess().getDescriptor().getOperator();
|
||||
log.info("Setting process operator {} in synchronization thread ",operator);
|
||||
Security.set(operator);
|
||||
|
||||
StorageHubClient client=WorkspaceUtils.getClient();
|
||||
FolderContainer parentFolder=client.open(parentFolderItem.getId()).asFolder();
|
||||
checkCancelledProcess();
|
||||
SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(parentFolder).getConfiguration();
|
||||
ThreddsController controller=new ThreddsController(synchConfig.getRemotePath());
|
||||
SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(parentFolder);
|
||||
ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), synchConfig.getTargetToken());
|
||||
|
||||
if(theRequest instanceof TransferToThreddsRequest) {
|
||||
TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest;
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.faults;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString(callSuper = true)
|
||||
public class GenericWebException extends Exception{
|
||||
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 5200927893712698886L;
|
||||
|
||||
private String remoteMessage=null;
|
||||
|
||||
private Integer responseHTTPCode=0;
|
||||
|
||||
public GenericWebException() {
|
||||
super();
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public GenericWebException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public GenericWebException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public GenericWebException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public GenericWebException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.model;
|
||||
|
||||
public enum ContainerType {
|
||||
|
||||
FOLDER,
|
||||
FILE,
|
||||
GENERIC_ITEM,
|
||||
URL
|
||||
|
||||
}
|
||||
|
|
@ -2,15 +2,13 @@ package org.gcube.usecases.ws.thredds.model;
|
|||
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString(callSuper=true)
|
||||
public class SyncFolderDescriptor extends SynchronizedElementInfo{
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
public class SyncFolderDescriptor {
|
||||
|
||||
@NonNull
|
||||
private String folderId;
|
||||
|
|
@ -18,27 +16,10 @@ public class SyncFolderDescriptor extends SynchronizedElementInfo{
|
|||
private String folderPath;
|
||||
@NonNull
|
||||
private SynchFolderConfiguration configuration;
|
||||
|
||||
@NonNull
|
||||
private Boolean isLocked=false;
|
||||
private boolean isLocked=false;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public SyncFolderDescriptor(SynchronizationStatus status, String folderId, String folderPath,
|
||||
SynchFolderConfiguration configuration, Boolean isLocked) {
|
||||
super(status);
|
||||
this.folderId = folderId;
|
||||
this.folderPath = folderPath;
|
||||
this.configuration = configuration;
|
||||
this.isLocked = isLocked;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private ProcessDescriptor localProcessDescriptor=null;
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,10 +16,8 @@ public class SynchFolderConfiguration {
|
|||
private String remotePath;
|
||||
@NonNull
|
||||
private String filter;
|
||||
|
||||
|
||||
@NonNull
|
||||
private String targetContext;
|
||||
private String targetToken;
|
||||
|
||||
|
||||
@NonNull
|
||||
|
|
|
|||
|
|
@ -1,20 +1,10 @@
|
|||
package org.gcube.usecases.ws.thredds.model;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@Data
|
||||
public class SynchronizedElementInfo {
|
||||
|
||||
public static enum SynchronizationStatus{
|
||||
UP_TO_DATE,OUTDATED_WS,OUTDATED_REMOTE
|
||||
}
|
||||
|
||||
@NonNull
|
||||
private SynchronizationStatus status;
|
||||
|
||||
private StepReport.Status lastSynchronizationStatus;
|
||||
private String lastupdateTime;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Root logger option
|
||||
log4j.rootLogger=DEBUG, SM
|
||||
#log4j.logger.org.gcube.usecases.ws=DEBUG,SM
|
||||
#log4j.rootLogger=INFO, SM
|
||||
log4j.logger.org.gcube.usecases.ws=DEBUG,SM
|
||||
|
||||
|
||||
|
||||
|
|
@ -14,13 +14,10 @@ import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
|||
public class DTSynchUseCase {
|
||||
|
||||
public static void main(String[] args) throws WorkspaceInteractionException, InternalException, ProcessNotFoundException {
|
||||
// TokenSetter.set("/d4science.research-infrastructures.eu");
|
||||
// String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d";
|
||||
|
||||
TokenSetter.set("/gcube/devsec/devVRE");
|
||||
String folderId="8ebe9ffb-e2cf-4b3e-ab91-cc6933d86625";
|
||||
TokenSetter.set("/d4science.research-infrastructures.eu");
|
||||
SyncEngine engine=SyncEngine.get();
|
||||
|
||||
String folderId="a8cd78d3-69e8-4d02-ac90-681b2d16d84d";
|
||||
|
||||
|
||||
// String folderId="8a6f9749-68d7-4a9a-a475-bd645050c3fd"; // sub folder for faster tests
|
||||
|
|
@ -35,7 +32,7 @@ public class DTSynchUseCase {
|
|||
|
||||
|
||||
SynchFolderConfiguration config=new SynchFolderConfiguration("public/netcdf/GPTest", "",
|
||||
"...", // devVRE
|
||||
"***REMOVED***", // devVRE
|
||||
"Agro",folderId);
|
||||
|
||||
engine.setSynchronizedFolder(config, folderId);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package org.gcube.usecases.ws.thredds;
|
|||
|
||||
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
||||
public class DTTests {
|
||||
|
|
@ -14,7 +13,7 @@ public class DTTests {
|
|||
StorageHubClient client=WorkspaceUtils.getClient();
|
||||
|
||||
String folderId=TestCommons.getByPath("/Workspace/Accounting").getId();
|
||||
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", Security.getCurrent().getContext(), "dummy",folderId), folderId);
|
||||
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId);
|
||||
System.out.println("Done");
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ public class FileAccessTests {
|
|||
String processID="stupidProcess";
|
||||
String toLockFolder="devVRE";
|
||||
|
||||
final ThreddsController controller=new ThreddsController(toLockFolder);
|
||||
final ThreddsController controller=new ThreddsController(toLockFolder, TokenSetter.getCurrentToken());
|
||||
|
||||
Runnable readCatalog=new Runnable() {@Override public void run() { try{
|
||||
controller.readThreddsFile("catalog.xml");
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
|||
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.faults.InternalException;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
|
|
@ -37,16 +37,24 @@ public class TestCommons {
|
|||
|
||||
private static Map<String,TestSet> configs=new HashMap<>();
|
||||
|
||||
private static String toUseConfig="simple";
|
||||
private static String toUseConfig="default";
|
||||
|
||||
|
||||
static {
|
||||
|
||||
// configs.put("GP", new TestSet("GPTests","/d4science.research-infrastructures.eu","a8cd78d3-69e8-4d02-ac90-681b2d16d84d","","",""));
|
||||
|
||||
configs.put("simple", new TestSet("Simple label ","/gcube/devsec/devVRE", "Test1","public/netcdf/simpleFolder","3aa85bbf-d5f3-4df4-ad03-4f8f788eba3d-98187548","simple"));
|
||||
// folderName="WS-Tests";
|
||||
|
||||
configs.put("root", new TestSet("Simple label ","/gcube", "Test2","public/netcdf/wrong","3aa85bbf-d5f3-4df4-ad03-4f8f788eba3d-98187548","Wrong"));
|
||||
|
||||
configs.put("simple", new TestSet("Simple label ","/gcube", "Test1","public/netcdf/simpleFolder","***REMOVED***","simple"));
|
||||
configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
|
||||
|
||||
configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
|
||||
|
||||
// configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main"));
|
||||
//
|
||||
//
|
||||
// configs.put("pre", new TestSet("Default Tests","/pred4s/preprod/preVRE","Thredds Synch","Fabio WS","91a8433b-9491-42c1-95e2-420245ef1943-980114272","SynchTestCatalog"));
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -107,11 +115,11 @@ public class TestCommons {
|
|||
|
||||
public static ThreddsController getThreddsController() throws Exception {
|
||||
SynchFolderConfiguration config=getSynchConfig();
|
||||
return new ThreddsController(config.getRemotePath());
|
||||
return new ThreddsController(config.getRemotePath(), config.getTargetToken());
|
||||
}
|
||||
|
||||
public static SynchFolderConfiguration getSynchConfig() throws Exception {
|
||||
TestSet set=configs.get(toUseConfig);
|
||||
return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", Security.getCurrent().getContext(),set.getToCreateCatalogName(),set.getFolderId());
|
||||
return new SynchFolderConfiguration(set.getRemotePath(), "*.nc,*.ncml,*.asc", set.getTargetToken(),set.getToCreateCatalogName(),set.getFolderId());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,23 @@ public class ThreddsCatalogTests {
|
|||
// SyncEngine engine=SyncEngine.get();
|
||||
//
|
||||
// String[] toCheckTokens=new String[] {
|
||||
// "..."
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// "***REMOVED***",
|
||||
// };
|
||||
//
|
||||
//
|
||||
|
|
@ -31,7 +47,7 @@ public class ThreddsCatalogTests {
|
|||
//
|
||||
|
||||
|
||||
ThreddsController controller=new ThreddsController("");
|
||||
ThreddsController controller=new ThreddsController("",TokenSetter.getCurrentToken());
|
||||
|
||||
ThreddsInfo info=controller.getThreddsInfo();
|
||||
String [] paths=new String[] {
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds;
|
||||
|
||||
public class TokenCheck {
|
||||
|
||||
//
|
||||
// public static void main (String[] args) {
|
||||
// System.out.println(Security.getContext("8e74a17c-92f1-405a-b591-3a6090066248-98187548"));
|
||||
// System.out.println(Security.getContext("0e2c7963-8d3e-4ea6-a56d-ffda530dd0fa-98187548"));
|
||||
// }
|
||||
//
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.User;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class TokenSetter {
|
||||
|
||||
private static Properties props=null;
|
||||
|
||||
|
||||
public static synchronized void set(String scope){
|
||||
if(props==null) {
|
||||
props=new Properties();
|
||||
try {
|
||||
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||
}
|
||||
}
|
||||
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
||||
Security.set(new User(null, null, props.getProperty(scope), scope));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -8,8 +8,6 @@ import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
|||
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
||||
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceFolderManager;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceLockedException;
|
||||
import org.gcube.usecases.ws.thredds.faults.WorkspaceNotSynchedException;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
||||
public class WorkspaceProperties {
|
||||
|
|
@ -22,11 +20,7 @@ public class WorkspaceProperties {
|
|||
// printProperties(folder);
|
||||
// System.out.println(new WorkspaceFolderManager(folder.getId()).isSynched());
|
||||
|
||||
try{
|
||||
SyncEngine.get().check(folder.getId(), true);
|
||||
}catch(WorkspaceLockedException e) {
|
||||
System.err.println("Workspace is locked.");
|
||||
}
|
||||
SyncEngine.get().check(folder.getId(), true);
|
||||
scanForPrint(folder);
|
||||
|
||||
|
||||
|
|
@ -72,7 +66,6 @@ public class WorkspaceProperties {
|
|||
}
|
||||
for(ItemContainer<?> item:folder.list().withMetadata().getContainers())
|
||||
if(item.getType().equals(ContainerType.FOLDER))scanForPrint((FolderContainer) item);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -87,11 +80,6 @@ public class WorkspaceProperties {
|
|||
if(entry.getValue()==null) System.out.print(entry.getKey()+" is null;");
|
||||
else System.out.print(entry.getKey()+" = "+entry.getValue()+";");
|
||||
}
|
||||
try{
|
||||
System.out.println(SyncEngine.parseInfo(map, org.gcube.usecases.ws.thredds.model.ContainerType.valueOf(item.getType().toString())));
|
||||
}catch(Throwable t) {
|
||||
System.err.println("ITEM ID "+item.getId()+t.getMessage());
|
||||
}
|
||||
}
|
||||
System.out.println();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package org.gcube.usecases.ws.thredds;
|
|||
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
|
||||
|
|
@ -21,7 +20,6 @@ public class WorkspaceSynchronizationTest {
|
|||
|
||||
//TEST INFO...
|
||||
TestCommons.setScope();
|
||||
System.out.println(ScopeProvider.instance.get());
|
||||
FolderContainer folder=TestCommons.getTestFolder();
|
||||
|
||||
// FOLDER CONFIGURATION BEAN
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
<configuration debug="true">
|
||||
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<!-- encoders are assigned the type
|
||||
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<root level="debug">
|
||||
<appender-ref ref="STDOUT" />
|
||||
</root>
|
||||
</configuration>
|
||||
Loading…
Reference in New Issue