test fixes
This commit is contained in:
parent
0c94a1578b
commit
5f72cc4323
|
@ -1 +1,2 @@
|
||||||
/org.eclipse.jdt.core.prefs
|
/org.eclipse.jdt.core.prefs
|
||||||
|
/org.eclipse.core.resources.prefs
|
||||||
|
|
|
@ -40,30 +40,31 @@ public class Commons {
|
||||||
|
|
||||||
|
|
||||||
public static void cleanupFolder(String toCleanPath, String destinationToken) {
|
public static void cleanupFolder(String toCleanPath, String destinationToken) {
|
||||||
String toRestoreToken=TokenSetter.getCurrentToken();
|
throw new RuntimeException("ILLEGAL OPERATION");
|
||||||
try{
|
// String toRestoreToken=TokenSetter.getCurrentToken();
|
||||||
log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath);
|
// try{
|
||||||
TokenSetter.setToken(destinationToken);
|
// log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath);
|
||||||
String hostname=getThreddsHost();
|
// TokenSetter.setGcubeToken(destinationToken);
|
||||||
DataTransferClient client=getDTClient(hostname);
|
// String hostname=getThreddsHost();
|
||||||
File toTransfer=File.createTempFile("clean", ".dt_temp");
|
// DataTransferClient client=getDTClient(hostname);
|
||||||
toTransfer.createNewFile();
|
// File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||||
Destination dest=new Destination();
|
// toTransfer.createNewFile();
|
||||||
dest.setCreateSubfolders(true);
|
// Destination dest=new Destination();
|
||||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
// dest.setCreateSubfolders(true);
|
||||||
dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
|
// dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||||
dest.setPersistenceId("thredds");
|
// dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
|
||||||
dest.setSubFolder("public/netcdf/"+toCleanPath);
|
// dest.setPersistenceId("thredds");
|
||||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
// dest.setSubFolder("public/netcdf/"+toCleanPath);
|
||||||
client.localFile(toTransfer, dest);
|
// log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
||||||
log.info("Done");
|
// client.localFile(toTransfer, dest);
|
||||||
}catch(Exception e) {
|
// log.info("Done");
|
||||||
log.error("Unable to delete remote folder "+toCleanPath,e);
|
// }catch(Exception e) {
|
||||||
throw new RuntimeException("Unable to cleanup remote folder.");
|
// log.error("Unable to delete remote folder "+toCleanPath,e);
|
||||||
}finally {
|
// throw new RuntimeException("Unable to cleanup remote folder.");
|
||||||
log.debug("Resetting original token {} ",toRestoreToken);
|
// }finally {
|
||||||
TokenSetter.set(toRestoreToken);
|
// log.debug("Resetting original token {} ",toRestoreToken);
|
||||||
}
|
// TokenSetter.set(toRestoreToken);
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,50 +0,0 @@
|
||||||
package org.gcube.usecases.ws.thredds;
|
|
||||||
|
|
||||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
|
||||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public class TokenSetter {
|
|
||||||
|
|
||||||
private static Properties props=null;
|
|
||||||
|
|
||||||
static{
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public static synchronized void set(String scope){
|
|
||||||
if(props==null) {
|
|
||||||
props=new Properties();
|
|
||||||
try {
|
|
||||||
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
|
||||||
SecurityTokenProvider.instance.set(props.getProperty(scope));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public static void setToken(String token){
|
|
||||||
try{
|
|
||||||
AuthorizationEntry entry = authorizationService().get(token);
|
|
||||||
ScopeProvider.instance.set(entry.getContext());
|
|
||||||
SecurityTokenProvider.instance.set(token);
|
|
||||||
}catch(Throwable t) {
|
|
||||||
throw new RuntimeException("Unable to set token "+token,t);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getCurrentToken() {
|
|
||||||
return SecurityTokenProvider.instance.get();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,208 +0,0 @@
|
||||||
package org.gcube.usecases.ws.thredds.engine;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.Semaphore;
|
|
||||||
|
|
||||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
|
||||||
import org.gcube.data.transfer.library.DataTransferClient;
|
|
||||||
import org.gcube.data.transfer.library.TransferResult;
|
|
||||||
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
|
||||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
|
||||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
|
||||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
|
||||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
|
||||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
|
||||||
import org.gcube.data.transfer.model.Destination;
|
|
||||||
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
|
||||||
import org.gcube.data.transfer.model.PluginInvocation;
|
|
||||||
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
|
|
||||||
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
|
|
||||||
import org.gcube.usecases.ws.thredds.Commons;
|
|
||||||
import org.gcube.usecases.ws.thredds.NetUtils;
|
|
||||||
import org.gcube.usecases.ws.thredds.TokenSetter;
|
|
||||||
import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
|
|
||||||
|
|
||||||
import lombok.NonNull;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
@Slf4j
|
|
||||||
public class PublishThread implements Runnable {
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@NonNull
|
|
||||||
private PublishRequest request;
|
|
||||||
@NonNull
|
|
||||||
private ConcurrentHashMap<String,PublishReport> reports;
|
|
||||||
|
|
||||||
|
|
||||||
private PublishReport publishReport;
|
|
||||||
// private Map<String,Report> reports;
|
|
||||||
//
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
log.info("Request is {}",request);
|
|
||||||
log.debug("Switching from {} to {}",SecurityTokenProvider.instance.get(),request.getPublishToken());
|
|
||||||
TokenSetter.setToken(request.getPublishToken());
|
|
||||||
log.debug("Current scope is :{}, token is {} ",ScopeUtils.getCurrentScope(),SecurityTokenProvider.instance.get());
|
|
||||||
|
|
||||||
Destination dest=new Destination();
|
|
||||||
dest.setPersistenceId("thredds");
|
|
||||||
dest.setSubFolder("public/netcdf/"+request.getCatalog());
|
|
||||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
|
||||||
dest.setCreateSubfolders(true);
|
|
||||||
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
|
||||||
String threddsHostName;
|
|
||||||
|
|
||||||
|
|
||||||
try {
|
|
||||||
threddsHostName = Commons.getThreddsHost();
|
|
||||||
|
|
||||||
DataTransferClient client=Commons.getDTClient(threddsHostName);
|
|
||||||
|
|
||||||
|
|
||||||
File toPublishSource=null;
|
|
||||||
|
|
||||||
if(request.getMode().equals(Mode.NCML)) {
|
|
||||||
if(request.isQueue()){
|
|
||||||
log.debug("Waiting for queue {}, expected Count {} ",request.getQueueId(),request.getQueueCount());
|
|
||||||
waitFor(request.getQueueId(), request.getQueueCount());
|
|
||||||
|
|
||||||
log.debug("Loading netcdfFile ..");
|
|
||||||
File ncmlFile=NetUtils.download(new URL(request.getSource().getUrl()));
|
|
||||||
String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath()));
|
|
||||||
|
|
||||||
for(String reportId:request.getToGatherReportsId()) {
|
|
||||||
PublishReport report=getReport(reportId);
|
|
||||||
//file://home/gcube/julien.barde/Workspace/DataMiner/Output_Data_Sets/Ichthyop2013.nc
|
|
||||||
|
|
||||||
String toSetUrl="file:/"+report.getTransferResult().getRemotePath();
|
|
||||||
toUpdateSource=toUpdateSource.replaceAll(reportId, toSetUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
toPublishSource=File.createTempFile("nc_", ".ncml");
|
|
||||||
PrintWriter out = new PrintWriter(toPublishSource);
|
|
||||||
out.write(toUpdateSource);
|
|
||||||
out.flush();
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
TransferResult result;
|
|
||||||
|
|
||||||
|
|
||||||
// TODO NB Test run without metadata publication
|
|
||||||
// result=client.httpSource(request.getSource().getUrl(), dest);
|
|
||||||
// publishReport=new PublishReport(false,request.getSource().getId(),result,null);
|
|
||||||
|
|
||||||
|
|
||||||
if(!request.isGenerateMeta()) {
|
|
||||||
log.debug("Transfering before publishing meta..");
|
|
||||||
|
|
||||||
result = toPublishSource==null?client.httpSource(request.getSource().getUrl(), dest):
|
|
||||||
client.localFile(toPublishSource, dest);
|
|
||||||
|
|
||||||
//NB DECOMMENT THIS!!!!!!
|
|
||||||
// Metadata meta=SDIAbstractPlugin.metadata().build();
|
|
||||||
//
|
|
||||||
// log.debug("Publishing metadata.. ");
|
|
||||||
//
|
|
||||||
// MetadataPublishOptions opts=new MetadataPublishOptions(new TemplateInvocationBuilder().threddsOnlineResources(threddsHostName, request.getSource().getName(), request.getCatalog()).get());
|
|
||||||
// opts.setGeonetworkCategory("Datasets");
|
|
||||||
// MetadataReport report=meta.pushMetadata(request.getMetadata(), opts);
|
|
||||||
|
|
||||||
// publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,report);
|
|
||||||
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,new MetadataReport());
|
|
||||||
|
|
||||||
|
|
||||||
}else {
|
|
||||||
log.debug("Metadata not provided.. ");
|
|
||||||
if(request.isQueue()&&request.getMode().equals(Mode.NC)) {
|
|
||||||
log.debug("Dataset file is linked in ncml, skipping metadata generation");
|
|
||||||
result=client.httpSource(request.getSource().getUrl(), dest);
|
|
||||||
}else
|
|
||||||
result=client.httpSource(request.getSource().getUrl(), dest,new PluginInvocation("SIS/GEOTK"));
|
|
||||||
|
|
||||||
|
|
||||||
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,null);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
} catch (UnreachableNodeException | ServiceNotFoundException e) {
|
|
||||||
log.error("Unable to find Thredds. Publish scope is {} ",ScopeUtils.getCurrentScope(),e);
|
|
||||||
} catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException
|
|
||||||
| InvalidDestinationException | DestinationNotSetException e) {
|
|
||||||
log.error("Unable to transfer file, ",e);
|
|
||||||
} catch (IOException e) {
|
|
||||||
log.error("Unable to read/ write file. ",e);
|
|
||||||
}
|
|
||||||
|
|
||||||
onCompletion();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void onCompletion() {
|
|
||||||
if(publishReport==null) publishReport=new PublishReport(true, request.getSource().getId(),request.getSource().getName(), null, null);
|
|
||||||
publishReport(publishReport);
|
|
||||||
|
|
||||||
if(request.getMode().equals(Mode.NC)&&(request.isQueue())) {
|
|
||||||
alert(request.getQueueId(),request.getQueueCount());
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private PublishReport getReport(String reportId) {
|
|
||||||
return reports.get(reportId);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void publishReport(PublishReport report) {
|
|
||||||
reports.put(report.getSourceId(), report);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// static
|
|
||||||
|
|
||||||
|
|
||||||
private static ConcurrentHashMap<String,Semaphore> semaphores=new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private static void waitFor(String queueId,Integer expected) {
|
|
||||||
try {
|
|
||||||
log.debug("Waiting for queue {}. Expected Count is {} ",queueId,expected);
|
|
||||||
semaphores.getOrDefault(queueId, new Semaphore(expected*-1)).acquire();
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
log.debug("Queue {} is completed.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void alert(String queueId, Integer expected) {
|
|
||||||
log.debug("Alerting queue {}. Expected count is {} ",queueId,expected);
|
|
||||||
Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1));
|
|
||||||
sem.release();
|
|
||||||
log.debug(String.format("Queue %1$s alerted. Remaining : %2$s out of %3$s ",queueId,sem.availablePermits(),expected));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,102 +0,0 @@
|
||||||
package org.gcube.usecases.ws.thredds.engine;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
import java.util.concurrent.BlockingQueue;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.LinkedBlockingDeque;
|
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
|
||||||
import org.gcube.usecases.ws.thredds.FolderConfiguration;
|
|
||||||
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.Synchronized;
|
|
||||||
|
|
||||||
public class TransferRequestServer {
|
|
||||||
|
|
||||||
@Data
|
|
||||||
public static class Report{
|
|
||||||
private AtomicLong requestCount=new AtomicLong(0l);
|
|
||||||
private AtomicLong requestServed=new AtomicLong(0l);
|
|
||||||
private ConcurrentHashMap<String,PublishReport> reports=new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
|
|
||||||
public File toFile(FolderConfiguration configuration) {
|
|
||||||
return toFile(configuration,this);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static final File toFile(FolderConfiguration config,Report report) {
|
|
||||||
PrintWriter writer =null;
|
|
||||||
try {
|
|
||||||
File toReturn=File.createTempFile("tempFile", ".tmp");
|
|
||||||
writer=new PrintWriter(toReturn);
|
|
||||||
|
|
||||||
writer.println("REPORT FOR WS-SYNCH");
|
|
||||||
writer.println("Configuratiion was : "+config);
|
|
||||||
writer.println("Submitted runs : "+report.getRequestCount());
|
|
||||||
writer.println("Item reports : ");
|
|
||||||
for(Entry<String,PublishReport> entry: report.getReports().entrySet()) {
|
|
||||||
PublishReport rep=entry.getValue();
|
|
||||||
writer.println("*********************************************************");
|
|
||||||
if(rep.isError()) writer.println("OPERATION IS FAILED");
|
|
||||||
writer.println("ITEM ID : "+rep.getSourceId());
|
|
||||||
writer.println("ITEM NAME : "+rep.getSourceName());
|
|
||||||
if(rep.getTransferResult()!=null)writer.println("Transfer report : "+rep.getTransferResult());
|
|
||||||
if(rep.getMetadataReport()!=null)writer.println("Metadata report : "+rep.getMetadataReport());
|
|
||||||
}
|
|
||||||
return toReturn;
|
|
||||||
}catch(Throwable t) {
|
|
||||||
throw new RuntimeException(t);
|
|
||||||
}finally {
|
|
||||||
if(writer!=null) {
|
|
||||||
IOUtils.closeQuietly(writer);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Report report=new Report();
|
|
||||||
private ExecutorService service=null;
|
|
||||||
|
|
||||||
public TransferRequestServer() {
|
|
||||||
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(
|
|
||||||
100);
|
|
||||||
service= new ThreadPoolExecutor(1, 10, 30,
|
|
||||||
TimeUnit.SECONDS, linkedBlockingDeque,
|
|
||||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void put(PublishRequest request){
|
|
||||||
System.out.println("Submitting transfer "+getReport().requestCount.incrementAndGet());
|
|
||||||
service.execute(new PublishThread(request, getReport().getReports()));
|
|
||||||
|
|
||||||
// service.execute(new RequestThread(baseUrl,filename,this,publishScope,toPublishMeta));
|
|
||||||
}
|
|
||||||
@Synchronized
|
|
||||||
public Report getReport(){
|
|
||||||
return report;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void waitCompletion() {
|
|
||||||
boolean running=true;
|
|
||||||
service.shutdown();
|
|
||||||
while(running){
|
|
||||||
System.out.println("******************* WAITING FOR TERMINATION ***************** ");
|
|
||||||
try{
|
|
||||||
running=!service.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
|
|
||||||
}catch(InterruptedException e){
|
|
||||||
running=!service.isTerminated();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
System.out.println("Service is completed : "+service.isTerminated());
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -66,19 +66,19 @@ public class ThreddsController {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private String hostname;
|
private URL threddsEndpoints;
|
||||||
private String operatingPath;
|
private String operatingPath;
|
||||||
|
|
||||||
public ThreddsController(String path) throws InternalException {
|
public ThreddsController(String path) throws InternalException {
|
||||||
operatingPath=path;
|
operatingPath=path;
|
||||||
|
|
||||||
hostname=getThreddsHost();
|
threddsEndpoints=getThreddsEndpoint();
|
||||||
if(hostname==null) throw new InternalException("Invalid hostnam in context "+ScopeUtils.getCurrentScope());
|
if(threddsEndpoints==null) throw new InternalException("Invalid hostnam in context "+ScopeUtils.getCurrentScope());
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final String truncate(String toTruncate) {
|
// private static final String truncate(String toTruncate) {
|
||||||
return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"...";
|
// return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"...";
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
// private void setTargetToken() {
|
// private void setTargetToken() {
|
||||||
|
@ -100,7 +100,7 @@ public class ThreddsController {
|
||||||
// }
|
// }
|
||||||
|
|
||||||
public final ThreddsInfo getThreddsInfo() {
|
public final ThreddsInfo getThreddsInfo() {
|
||||||
String infoPath="https://"+hostname+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";;
|
String infoPath="https://"+threddsEndpoints.getHost()+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";;
|
||||||
log.info("Loading thredds info from {} ",infoPath);
|
log.info("Loading thredds info from {} ",infoPath);
|
||||||
WebTarget target=getWebClient().target(infoPath);
|
WebTarget target=getWebClient().target(infoPath);
|
||||||
return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class);
|
return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class);
|
||||||
|
@ -112,7 +112,7 @@ public class ThreddsController {
|
||||||
File temp=null;
|
File temp=null;
|
||||||
try{
|
try{
|
||||||
log.info("Locking remote path {} to processId {} ",operatingPath,processId);
|
log.info("Locking remote path {} to processId {} ",operatingPath,processId);
|
||||||
DataTransferClient cl=getDTClient(hostname);
|
DataTransferClient cl=getDTClient(threddsEndpoints);
|
||||||
|
|
||||||
Destination dest=new Destination();
|
Destination dest=new Destination();
|
||||||
dest.setCreateSubfolders(true);
|
dest.setCreateSubfolders(true);
|
||||||
|
@ -143,7 +143,7 @@ public class ThreddsController {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteThreddsFile(String location) throws RemoteFileNotFoundException {
|
public void deleteThreddsFile(String location) throws RemoteFileNotFoundException {
|
||||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||||
log.info("Deleting file at {} ",urlString);
|
log.info("Deleting file at {} ",urlString);
|
||||||
try{
|
try{
|
||||||
getWebClient().target(urlString).request().delete();
|
getWebClient().target(urlString).request().delete();
|
||||||
|
@ -163,7 +163,7 @@ public class ThreddsController {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String readThreddsFile(String location) throws RemoteFileNotFoundException {
|
public String readThreddsFile(String location) throws RemoteFileNotFoundException {
|
||||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||||
log.info("Reading file at {} ",urlString);
|
log.info("Reading file at {} ",urlString);
|
||||||
try{
|
try{
|
||||||
return getWebClient().target(urlString).request().get().readEntity(String.class);
|
return getWebClient().target(urlString).request().get().readEntity(String.class);
|
||||||
|
@ -176,8 +176,8 @@ public class ThreddsController {
|
||||||
public void createEmptyFolder(String targetPath) throws InternalException {
|
public void createEmptyFolder(String targetPath) throws InternalException {
|
||||||
String toCleanPath=getPathFromStartingLocation(targetPath);
|
String toCleanPath=getPathFromStartingLocation(targetPath);
|
||||||
try{
|
try{
|
||||||
log.info("Cleaning up {} on {} ",toCleanPath,hostname);
|
log.info("Cleaning up {} on {} ",toCleanPath,threddsEndpoints);
|
||||||
DataTransferClient client=getDTClient(hostname);
|
DataTransferClient client=getDTClient(threddsEndpoints);
|
||||||
File toTransfer=File.createTempFile("clean", ".dt_temp");
|
File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||||
toTransfer.createNewFile();
|
toTransfer.createNewFile();
|
||||||
Destination dest=new Destination();
|
Destination dest=new Destination();
|
||||||
|
@ -188,7 +188,7 @@ public class ThreddsController {
|
||||||
dest.setSubFolder(toCleanPath);
|
dest.setSubFolder(toCleanPath);
|
||||||
dest.setDestinationFileName(toTransfer.getName());
|
dest.setDestinationFileName(toTransfer.getName());
|
||||||
|
|
||||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),threddsEndpoints);
|
||||||
client.localFile(toTransfer, dest);
|
client.localFile(toTransfer, dest);
|
||||||
this.deleteThreddsFile(targetPath+"/"+toTransfer.getName());
|
this.deleteThreddsFile(targetPath+"/"+toTransfer.getName());
|
||||||
log.info("Done");
|
log.info("Done");
|
||||||
|
@ -229,7 +229,7 @@ public class ThreddsController {
|
||||||
}
|
}
|
||||||
|
|
||||||
public RemoteFileDescriptor getFileDescriptor(String path) throws RemoteFileNotFoundException {
|
public RemoteFileDescriptor getFileDescriptor(String path) throws RemoteFileNotFoundException {
|
||||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||||
log.info("Reading file at {} ",urlString);
|
log.info("Reading file at {} ",urlString);
|
||||||
try{
|
try{
|
||||||
return getWebClient().target(urlString).queryParam("descriptor", true).request().get().readEntity(RemoteFileDescriptor.class);
|
return getWebClient().target(urlString).queryParam("descriptor", true).request().get().readEntity(RemoteFileDescriptor.class);
|
||||||
|
@ -239,7 +239,7 @@ public class ThreddsController {
|
||||||
}
|
}
|
||||||
|
|
||||||
public InputStream getInputStream(String path) throws RemoteFileNotFoundException {
|
public InputStream getInputStream(String path) throws RemoteFileNotFoundException {
|
||||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||||
log.info("Reading file at {} ",urlString);
|
log.info("Reading file at {} ",urlString);
|
||||||
try{
|
try{
|
||||||
return getWebClient().target(urlString).request().get().readEntity(InputStream.class);
|
return getWebClient().target(urlString).request().get().readEntity(InputStream.class);
|
||||||
|
@ -250,7 +250,7 @@ public class ThreddsController {
|
||||||
|
|
||||||
|
|
||||||
public TransferResult transferFile(Destination dest,URL url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
|
public TransferResult transferFile(Destination dest,URL url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
|
||||||
DataTransferClient client=getDTClient(hostname);
|
DataTransferClient client=getDTClient(threddsEndpoints);
|
||||||
if(invocations!=null&&!invocations.isEmpty())
|
if(invocations!=null&&!invocations.isEmpty())
|
||||||
return client.httpSource(url, dest,invocations);
|
return client.httpSource(url, dest,invocations);
|
||||||
else return client.httpSource(url, dest);
|
else return client.httpSource(url, dest);
|
||||||
|
@ -268,7 +268,7 @@ public class ThreddsController {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private String getThreddsHost() throws InternalException{
|
private URL getThreddsEndpoint() throws InternalException{
|
||||||
|
|
||||||
try{
|
try{
|
||||||
String sdiUrl="https://"+getSDIServiceHost()+"/"+Constants.SDI_CONFIG_PATH;
|
String sdiUrl="https://"+getSDIServiceHost()+"/"+Constants.SDI_CONFIG_PATH;
|
||||||
|
@ -279,7 +279,7 @@ public class ThreddsController {
|
||||||
throw new InternalException("Failed to contact SDI. Message "+respString);
|
throw new InternalException("Failed to contact SDI. Message "+respString);
|
||||||
|
|
||||||
DocumentContext sourceCtx=JsonPath.using(JSON_PATH_ALWAYS_LIST_CONFIG).parse(respString);
|
DocumentContext sourceCtx=JsonPath.using(JSON_PATH_ALWAYS_LIST_CONFIG).parse(respString);
|
||||||
return new URL(((List<String>) sourceCtx.read("$.threddsConfiguration..baseEndpoint")).get(0)).getHost();
|
return new URL(((List<String>) sourceCtx.read("$.threddsConfiguration..baseEndpoint")).get(0));
|
||||||
}catch(Throwable t) {
|
}catch(Throwable t) {
|
||||||
log.error("Unable to read SDI configuration",t);
|
log.error("Unable to read SDI configuration",t);
|
||||||
throw new InternalException("Unable to gt configuration from SDI",t);
|
throw new InternalException("Unable to gt configuration from SDI",t);
|
||||||
|
@ -304,8 +304,8 @@ public class ThreddsController {
|
||||||
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
|
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static DataTransferClient getDTClient(String threddsHostName) throws UnreachableNodeException, ServiceNotFoundException {
|
private static DataTransferClient getDTClient(URL threddsEndpoint) throws UnreachableNodeException, ServiceNotFoundException {
|
||||||
log.debug("Getting DT Client for {} ",threddsHostName);
|
log.debug("Getting DT Client for {} ",threddsEndpoint);
|
||||||
return DataTransferClient.getInstanceByEndpoint("https://"+threddsHostName);
|
return DataTransferClient.getInstanceByEndpoint(threddsEndpoint.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,9 @@
|
||||||
package org.gcube.usecases.ws.thredds.engine.impl.security;
|
package org.gcube.usecases.ws.thredds.engine.impl.security;
|
||||||
|
|
||||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
|
||||||
|
|
||||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
|
||||||
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
|
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
|
||||||
import org.gcube.common.authorization.library.provider.ClientInfo;
|
import org.gcube.common.authorization.library.provider.ClientInfo;
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
import org.gcube.common.authorization.library.provider.UmaJWTProvider;
|
import org.gcube.common.authorization.library.provider.UmaJWTProvider;
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||||
|
@ -17,17 +15,30 @@ public class Security {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public static User getCurrent() {
|
public static User getCurrent() throws SecurityException {
|
||||||
ClientInfo client = AuthorizationProvider.instance.get().getClient();
|
String context=ScopeProvider.instance.get();
|
||||||
User toReturn = new User(client,UmaJWTProvider.instance.get(),ScopeProvider.instance.get());
|
if(context==null) throw new SecurityException("Cannot determine context");
|
||||||
log.debug("Current User is {} ",toReturn);
|
|
||||||
|
log.debug("Context is {}, checking tokens..",context);
|
||||||
|
|
||||||
|
ClientInfo client = null;
|
||||||
|
try{
|
||||||
|
AuthorizationProvider.instance.get().getClient();
|
||||||
|
}catch(Exception e) {
|
||||||
|
log.warn("Unable to get client info ",e);
|
||||||
|
}
|
||||||
|
User toReturn = new User(client,UmaJWTProvider.instance.get(),SecurityTokenProvider.instance.get(),context);
|
||||||
|
|
||||||
|
log.info("Current User is {} ",toReturn);
|
||||||
return toReturn;
|
return toReturn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public static void set(User toSet) {
|
public static void set(User toSet) {
|
||||||
log.debug("Setting User {} ",toSet);
|
log.debug("Setting User {} ",toSet);
|
||||||
UmaJWTProvider.instance.set(toSet.getToken());
|
if(toSet.getUma_token()!=null)UmaJWTProvider.instance.set(toSet.getUma_token());
|
||||||
|
if(toSet.getGcube_token()!=null)UmaJWTProvider.instance.set(toSet.getUma_token());
|
||||||
|
if(ScopeProvider.instance.get()==null)ScopeProvider.instance.set(toSet.getContext());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void checkOperator(SynchFolderConfiguration config) throws SecurityException{
|
public static void checkOperator(SynchFolderConfiguration config) throws SecurityException{
|
||||||
|
@ -48,8 +59,15 @@ public class Security {
|
||||||
|
|
||||||
public static String getContextFromgcubeToken(String token) throws ObjectNotFound, Exception {
|
public static String getContextFromgcubeToken(String token) throws ObjectNotFound, Exception {
|
||||||
log.debug("Checking context of gcube-token {}...",token.substring(0,6));
|
log.debug("Checking context of gcube-token {}...",token.substring(0,6));
|
||||||
AuthorizationEntry entry = authorizationService().get(token);
|
User caller=getCurrent();
|
||||||
String context=entry.getContext();
|
try {
|
||||||
return context;
|
SecurityTokenProvider.instance.reset();
|
||||||
|
UmaJWTProvider.instance.reset();
|
||||||
|
SecurityTokenProvider.instance.set(token);
|
||||||
|
return ScopeProvider.instance.get();
|
||||||
|
}finally {
|
||||||
|
log.debug("Resetting user "+caller);
|
||||||
|
set(caller);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,9 @@ import lombok.Getter;
|
||||||
@Getter
|
@Getter
|
||||||
public class User{
|
public class User{
|
||||||
private ClientInfo user;
|
private ClientInfo user;
|
||||||
private String token;
|
private String uma_token;
|
||||||
|
|
||||||
|
private String gcube_token;
|
||||||
|
|
||||||
private String context;
|
private String context;
|
||||||
|
|
||||||
|
@ -18,8 +20,12 @@ public class User{
|
||||||
StringBuilder builder = new StringBuilder();
|
StringBuilder builder = new StringBuilder();
|
||||||
builder.append("User [user=");
|
builder.append("User [user=");
|
||||||
builder.append(user);
|
builder.append(user);
|
||||||
builder.append(", token=");
|
builder.append(", uma_token=");
|
||||||
builder.append(token==null?token:"***");
|
builder.append(uma_token==null?uma_token:"***");
|
||||||
|
|
||||||
|
builder.append(", gcube_token=");
|
||||||
|
builder.append(gcube_token==null?gcube_token:"***");
|
||||||
|
|
||||||
builder.append(", context=");
|
builder.append(", context=");
|
||||||
builder.append(context);
|
builder.append(context);
|
||||||
builder.append("]");
|
builder.append("]");
|
||||||
|
|
|
@ -2,6 +2,7 @@ package org.gcube.usecases.ws.thredds;
|
||||||
|
|
||||||
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
||||||
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
||||||
|
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||||
|
|
||||||
public class DTTests {
|
public class DTTests {
|
||||||
|
@ -13,7 +14,7 @@ public class DTTests {
|
||||||
StorageHubClient client=WorkspaceUtils.getClient();
|
StorageHubClient client=WorkspaceUtils.getClient();
|
||||||
|
|
||||||
String folderId=TestCommons.getByPath("/Workspace/Accounting").getId();
|
String folderId=TestCommons.getByPath("/Workspace/Accounting").getId();
|
||||||
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId);
|
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", Security.getCurrent().getContext(), "dummy",folderId), folderId);
|
||||||
System.out.println("Done");
|
System.out.println("Done");
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,10 +9,10 @@ public class GetTrhreddsInfoTest {
|
||||||
SyncEngine engine=SyncEngine.get();
|
SyncEngine engine=SyncEngine.get();
|
||||||
// ROOT
|
// ROOT
|
||||||
|
|
||||||
TokenSetter.setToken("***REMOVED***");
|
TokenSetter.setGcubeToken("***REMOVED***");
|
||||||
System.out.println(engine.getAvailableCatalogs());
|
System.out.println(engine.getAvailableCatalogs());
|
||||||
// MEI
|
// MEI
|
||||||
TokenSetter.setToken("54f577de-d259-407e-b30d-29bf9e7c0dee-843339462");
|
TokenSetter.setGcubeToken("54f577de-d259-407e-b30d-29bf9e7c0dee-843339462");
|
||||||
System.out.println(engine.getAvailableCatalogs());
|
System.out.println(engine.getAvailableCatalogs());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -46,11 +46,11 @@ public class TestCommons {
|
||||||
|
|
||||||
// folderName="WS-Tests";
|
// folderName="WS-Tests";
|
||||||
|
|
||||||
configs.put("simple", new TestSet("Simple label ","/gcube/devsec/devVRE", "Test1","public/netcdf/simpleFolder","***REMOVED***","simple"));
|
configs.put("simple", new TestSet("Simple label ","/gcube/devsec/devVRE", "Test1","public/netcdf/simpleFolder","3aa85bbf-d5f3-4df4-ad03-4f8f788eba3d-98187548","simple"));
|
||||||
configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
|
// configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
|
||||||
|
//
|
||||||
configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
|
// configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
|
||||||
|
//
|
||||||
// configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main"));
|
// configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main"));
|
||||||
//
|
//
|
||||||
//
|
//
|
||||||
|
|
|
@ -0,0 +1,30 @@
|
||||||
|
package org.gcube.usecases.ws.thredds;
|
||||||
|
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||||
|
import org.gcube.usecases.ws.thredds.engine.impl.security.User;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class TokenSetter {
|
||||||
|
|
||||||
|
private static Properties props=null;
|
||||||
|
|
||||||
|
|
||||||
|
public static synchronized void set(String scope){
|
||||||
|
if(props==null) {
|
||||||
|
props=new Properties();
|
||||||
|
try {
|
||||||
|
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
||||||
|
Security.set(new User(null, null, props.getProperty(scope), scope));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -2,6 +2,7 @@ package org.gcube.usecases.ws.thredds;
|
||||||
|
|
||||||
import java.util.concurrent.Semaphore;
|
import java.util.concurrent.Semaphore;
|
||||||
|
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
|
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
|
||||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
|
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
|
||||||
|
@ -20,6 +21,7 @@ public class WorkspaceSynchronizationTest {
|
||||||
|
|
||||||
//TEST INFO...
|
//TEST INFO...
|
||||||
TestCommons.setScope();
|
TestCommons.setScope();
|
||||||
|
System.out.println(ScopeProvider.instance.get());
|
||||||
FolderContainer folder=TestCommons.getTestFolder();
|
FolderContainer folder=TestCommons.getTestFolder();
|
||||||
|
|
||||||
// FOLDER CONFIGURATION BEAN
|
// FOLDER CONFIGURATION BEAN
|
||||||
|
|
Loading…
Reference in New Issue