test fixes
This commit is contained in:
parent
0c94a1578b
commit
5f72cc4323
|
@ -1 +1,2 @@
|
|||
/org.eclipse.jdt.core.prefs
|
||||
/org.eclipse.core.resources.prefs
|
||||
|
|
|
@ -40,30 +40,31 @@ public class Commons {
|
|||
|
||||
|
||||
public static void cleanupFolder(String toCleanPath, String destinationToken) {
|
||||
String toRestoreToken=TokenSetter.getCurrentToken();
|
||||
try{
|
||||
log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath);
|
||||
TokenSetter.setToken(destinationToken);
|
||||
String hostname=getThreddsHost();
|
||||
DataTransferClient client=getDTClient(hostname);
|
||||
File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||
toTransfer.createNewFile();
|
||||
Destination dest=new Destination();
|
||||
dest.setCreateSubfolders(true);
|
||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
|
||||
dest.setPersistenceId("thredds");
|
||||
dest.setSubFolder("public/netcdf/"+toCleanPath);
|
||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
||||
client.localFile(toTransfer, dest);
|
||||
log.info("Done");
|
||||
}catch(Exception e) {
|
||||
log.error("Unable to delete remote folder "+toCleanPath,e);
|
||||
throw new RuntimeException("Unable to cleanup remote folder.");
|
||||
}finally {
|
||||
log.debug("Resetting original token {} ",toRestoreToken);
|
||||
TokenSetter.set(toRestoreToken);
|
||||
}
|
||||
throw new RuntimeException("ILLEGAL OPERATION");
|
||||
// String toRestoreToken=TokenSetter.getCurrentToken();
|
||||
// try{
|
||||
// log.debug("Setting target token {} for cleanup request of path {} ",destinationToken,toCleanPath);
|
||||
// TokenSetter.setGcubeToken(destinationToken);
|
||||
// String hostname=getThreddsHost();
|
||||
// DataTransferClient client=getDTClient(hostname);
|
||||
// File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||
// toTransfer.createNewFile();
|
||||
// Destination dest=new Destination();
|
||||
// dest.setCreateSubfolders(true);
|
||||
// dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
// dest.setOnExistingSubFolder(DestinationClashPolicy.REWRITE);
|
||||
// dest.setPersistenceId("thredds");
|
||||
// dest.setSubFolder("public/netcdf/"+toCleanPath);
|
||||
// log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
||||
// client.localFile(toTransfer, dest);
|
||||
// log.info("Done");
|
||||
// }catch(Exception e) {
|
||||
// log.error("Unable to delete remote folder "+toCleanPath,e);
|
||||
// throw new RuntimeException("Unable to cleanup remote folder.");
|
||||
// }finally {
|
||||
// log.debug("Resetting original token {} ",toRestoreToken);
|
||||
// TokenSetter.set(toRestoreToken);
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -1,50 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class TokenSetter {
|
||||
|
||||
private static Properties props=null;
|
||||
|
||||
static{
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static synchronized void set(String scope){
|
||||
if(props==null) {
|
||||
props=new Properties();
|
||||
try {
|
||||
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||
}
|
||||
}
|
||||
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
||||
SecurityTokenProvider.instance.set(props.getProperty(scope));
|
||||
}
|
||||
|
||||
|
||||
public static void setToken(String token){
|
||||
try{
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
ScopeProvider.instance.set(entry.getContext());
|
||||
SecurityTokenProvider.instance.set(token);
|
||||
}catch(Throwable t) {
|
||||
throw new RuntimeException("Unable to set token "+token,t);
|
||||
}
|
||||
}
|
||||
|
||||
public static String getCurrentToken() {
|
||||
return SecurityTokenProvider.instance.get();
|
||||
}
|
||||
}
|
|
@ -1,208 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.engine;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.data.transfer.library.DataTransferClient;
|
||||
import org.gcube.data.transfer.library.TransferResult;
|
||||
import org.gcube.data.transfer.library.faults.DestinationNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.FailedTransferException;
|
||||
import org.gcube.data.transfer.library.faults.InitializationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidDestinationException;
|
||||
import org.gcube.data.transfer.library.faults.InvalidSourceException;
|
||||
import org.gcube.data.transfer.library.faults.ServiceNotFoundException;
|
||||
import org.gcube.data.transfer.library.faults.SourceNotSetException;
|
||||
import org.gcube.data.transfer.library.faults.UnreachableNodeException;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
|
||||
import org.gcube.spatial.data.sdi.utils.ScopeUtils;
|
||||
import org.gcube.usecases.ws.thredds.Commons;
|
||||
import org.gcube.usecases.ws.thredds.NetUtils;
|
||||
import org.gcube.usecases.ws.thredds.TokenSetter;
|
||||
import org.gcube.usecases.ws.thredds.engine.PublishRequest.Mode;
|
||||
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PublishThread implements Runnable {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@NonNull
|
||||
private PublishRequest request;
|
||||
@NonNull
|
||||
private ConcurrentHashMap<String,PublishReport> reports;
|
||||
|
||||
|
||||
private PublishReport publishReport;
|
||||
// private Map<String,Report> reports;
|
||||
//
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
log.info("Request is {}",request);
|
||||
log.debug("Switching from {} to {}",SecurityTokenProvider.instance.get(),request.getPublishToken());
|
||||
TokenSetter.setToken(request.getPublishToken());
|
||||
log.debug("Current scope is :{}, token is {} ",ScopeUtils.getCurrentScope(),SecurityTokenProvider.instance.get());
|
||||
|
||||
Destination dest=new Destination();
|
||||
dest.setPersistenceId("thredds");
|
||||
dest.setSubFolder("public/netcdf/"+request.getCatalog());
|
||||
dest.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
||||
dest.setCreateSubfolders(true);
|
||||
dest.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
||||
String threddsHostName;
|
||||
|
||||
|
||||
try {
|
||||
threddsHostName = Commons.getThreddsHost();
|
||||
|
||||
DataTransferClient client=Commons.getDTClient(threddsHostName);
|
||||
|
||||
|
||||
File toPublishSource=null;
|
||||
|
||||
if(request.getMode().equals(Mode.NCML)) {
|
||||
if(request.isQueue()){
|
||||
log.debug("Waiting for queue {}, expected Count {} ",request.getQueueId(),request.getQueueCount());
|
||||
waitFor(request.getQueueId(), request.getQueueCount());
|
||||
|
||||
log.debug("Loading netcdfFile ..");
|
||||
File ncmlFile=NetUtils.download(new URL(request.getSource().getUrl()));
|
||||
String toUpdateSource=new String(Files.readAllBytes(ncmlFile.toPath()));
|
||||
|
||||
for(String reportId:request.getToGatherReportsId()) {
|
||||
PublishReport report=getReport(reportId);
|
||||
//file://home/gcube/julien.barde/Workspace/DataMiner/Output_Data_Sets/Ichthyop2013.nc
|
||||
|
||||
String toSetUrl="file:/"+report.getTransferResult().getRemotePath();
|
||||
toUpdateSource=toUpdateSource.replaceAll(reportId, toSetUrl);
|
||||
}
|
||||
|
||||
toPublishSource=File.createTempFile("nc_", ".ncml");
|
||||
PrintWriter out = new PrintWriter(toPublishSource);
|
||||
out.write(toUpdateSource);
|
||||
out.flush();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
TransferResult result;
|
||||
|
||||
|
||||
// TODO NB Test run without metadata publication
|
||||
// result=client.httpSource(request.getSource().getUrl(), dest);
|
||||
// publishReport=new PublishReport(false,request.getSource().getId(),result,null);
|
||||
|
||||
|
||||
if(!request.isGenerateMeta()) {
|
||||
log.debug("Transfering before publishing meta..");
|
||||
|
||||
result = toPublishSource==null?client.httpSource(request.getSource().getUrl(), dest):
|
||||
client.localFile(toPublishSource, dest);
|
||||
|
||||
//NB DECOMMENT THIS!!!!!!
|
||||
// Metadata meta=SDIAbstractPlugin.metadata().build();
|
||||
//
|
||||
// log.debug("Publishing metadata.. ");
|
||||
//
|
||||
// MetadataPublishOptions opts=new MetadataPublishOptions(new TemplateInvocationBuilder().threddsOnlineResources(threddsHostName, request.getSource().getName(), request.getCatalog()).get());
|
||||
// opts.setGeonetworkCategory("Datasets");
|
||||
// MetadataReport report=meta.pushMetadata(request.getMetadata(), opts);
|
||||
|
||||
// publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,report);
|
||||
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,new MetadataReport());
|
||||
|
||||
|
||||
}else {
|
||||
log.debug("Metadata not provided.. ");
|
||||
if(request.isQueue()&&request.getMode().equals(Mode.NC)) {
|
||||
log.debug("Dataset file is linked in ncml, skipping metadata generation");
|
||||
result=client.httpSource(request.getSource().getUrl(), dest);
|
||||
}else
|
||||
result=client.httpSource(request.getSource().getUrl(), dest,new PluginInvocation("SIS/GEOTK"));
|
||||
|
||||
|
||||
publishReport=new PublishReport(false,request.getSource().getId(),request.getSource().getName(),result,null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} catch (UnreachableNodeException | ServiceNotFoundException e) {
|
||||
log.error("Unable to find Thredds. Publish scope is {} ",ScopeUtils.getCurrentScope(),e);
|
||||
} catch (InvalidSourceException | SourceNotSetException | FailedTransferException | InitializationException
|
||||
| InvalidDestinationException | DestinationNotSetException e) {
|
||||
log.error("Unable to transfer file, ",e);
|
||||
} catch (IOException e) {
|
||||
log.error("Unable to read/ write file. ",e);
|
||||
}
|
||||
|
||||
onCompletion();
|
||||
}
|
||||
|
||||
|
||||
private void onCompletion() {
|
||||
if(publishReport==null) publishReport=new PublishReport(true, request.getSource().getId(),request.getSource().getName(), null, null);
|
||||
publishReport(publishReport);
|
||||
|
||||
if(request.getMode().equals(Mode.NC)&&(request.isQueue())) {
|
||||
alert(request.getQueueId(),request.getQueueCount());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private PublishReport getReport(String reportId) {
|
||||
return reports.get(reportId);
|
||||
}
|
||||
|
||||
|
||||
private void publishReport(PublishReport report) {
|
||||
reports.put(report.getSourceId(), report);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// static
|
||||
|
||||
|
||||
private static ConcurrentHashMap<String,Semaphore> semaphores=new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
|
||||
private static void waitFor(String queueId,Integer expected) {
|
||||
try {
|
||||
log.debug("Waiting for queue {}. Expected Count is {} ",queueId,expected);
|
||||
semaphores.getOrDefault(queueId, new Semaphore(expected*-1)).acquire();
|
||||
} catch (InterruptedException e) {
|
||||
log.debug("Queue {} is completed.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void alert(String queueId, Integer expected) {
|
||||
log.debug("Alerting queue {}. Expected count is {} ",queueId,expected);
|
||||
Semaphore sem=semaphores.getOrDefault(queueId, new Semaphore(expected*-1));
|
||||
sem.release();
|
||||
log.debug(String.format("Queue %1$s alerted. Remaining : %2$s out of %3$s ",queueId,sem.availablePermits(),expected));
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
|
@ -1,102 +0,0 @@
|
|||
package org.gcube.usecases.ws.thredds.engine;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.gcube.usecases.ws.thredds.FolderConfiguration;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Synchronized;
|
||||
|
||||
public class TransferRequestServer {
|
||||
|
||||
@Data
|
||||
public static class Report{
|
||||
private AtomicLong requestCount=new AtomicLong(0l);
|
||||
private AtomicLong requestServed=new AtomicLong(0l);
|
||||
private ConcurrentHashMap<String,PublishReport> reports=new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
public File toFile(FolderConfiguration configuration) {
|
||||
return toFile(configuration,this);
|
||||
}
|
||||
|
||||
|
||||
private static final File toFile(FolderConfiguration config,Report report) {
|
||||
PrintWriter writer =null;
|
||||
try {
|
||||
File toReturn=File.createTempFile("tempFile", ".tmp");
|
||||
writer=new PrintWriter(toReturn);
|
||||
|
||||
writer.println("REPORT FOR WS-SYNCH");
|
||||
writer.println("Configuratiion was : "+config);
|
||||
writer.println("Submitted runs : "+report.getRequestCount());
|
||||
writer.println("Item reports : ");
|
||||
for(Entry<String,PublishReport> entry: report.getReports().entrySet()) {
|
||||
PublishReport rep=entry.getValue();
|
||||
writer.println("*********************************************************");
|
||||
if(rep.isError()) writer.println("OPERATION IS FAILED");
|
||||
writer.println("ITEM ID : "+rep.getSourceId());
|
||||
writer.println("ITEM NAME : "+rep.getSourceName());
|
||||
if(rep.getTransferResult()!=null)writer.println("Transfer report : "+rep.getTransferResult());
|
||||
if(rep.getMetadataReport()!=null)writer.println("Metadata report : "+rep.getMetadataReport());
|
||||
}
|
||||
return toReturn;
|
||||
}catch(Throwable t) {
|
||||
throw new RuntimeException(t);
|
||||
}finally {
|
||||
if(writer!=null) {
|
||||
IOUtils.closeQuietly(writer);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Report report=new Report();
|
||||
private ExecutorService service=null;
|
||||
|
||||
public TransferRequestServer() {
|
||||
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(
|
||||
100);
|
||||
service= new ThreadPoolExecutor(1, 10, 30,
|
||||
TimeUnit.SECONDS, linkedBlockingDeque,
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
|
||||
public void put(PublishRequest request){
|
||||
System.out.println("Submitting transfer "+getReport().requestCount.incrementAndGet());
|
||||
service.execute(new PublishThread(request, getReport().getReports()));
|
||||
|
||||
// service.execute(new RequestThread(baseUrl,filename,this,publishScope,toPublishMeta));
|
||||
}
|
||||
@Synchronized
|
||||
public Report getReport(){
|
||||
return report;
|
||||
}
|
||||
|
||||
|
||||
public void waitCompletion() {
|
||||
boolean running=true;
|
||||
service.shutdown();
|
||||
while(running){
|
||||
System.out.println("******************* WAITING FOR TERMINATION ***************** ");
|
||||
try{
|
||||
running=!service.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
|
||||
}catch(InterruptedException e){
|
||||
running=!service.isTerminated();
|
||||
}
|
||||
}
|
||||
System.out.println("Service is completed : "+service.isTerminated());
|
||||
}
|
||||
|
||||
}
|
|
@ -66,19 +66,19 @@ public class ThreddsController {
|
|||
|
||||
|
||||
|
||||
private String hostname;
|
||||
private URL threddsEndpoints;
|
||||
private String operatingPath;
|
||||
|
||||
public ThreddsController(String path) throws InternalException {
|
||||
operatingPath=path;
|
||||
|
||||
hostname=getThreddsHost();
|
||||
if(hostname==null) throw new InternalException("Invalid hostnam in context "+ScopeUtils.getCurrentScope());
|
||||
threddsEndpoints=getThreddsEndpoint();
|
||||
if(threddsEndpoints==null) throw new InternalException("Invalid hostnam in context "+ScopeUtils.getCurrentScope());
|
||||
}
|
||||
|
||||
private static final String truncate(String toTruncate) {
|
||||
return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"...";
|
||||
}
|
||||
// private static final String truncate(String toTruncate) {
|
||||
// return toTruncate==null?toTruncate:toTruncate.substring(0, toTruncate.length()/2)+"...";
|
||||
// }
|
||||
|
||||
|
||||
// private void setTargetToken() {
|
||||
|
@ -100,7 +100,7 @@ public class ThreddsController {
|
|||
// }
|
||||
|
||||
public final ThreddsInfo getThreddsInfo() {
|
||||
String infoPath="https://"+hostname+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";;
|
||||
String infoPath="https://"+threddsEndpoints.getHost()+"/data-transfer-service/gcube/service/Capabilities/pluginInfo/REGISTER_CATALOG";;
|
||||
log.info("Loading thredds info from {} ",infoPath);
|
||||
WebTarget target=getWebClient().target(infoPath);
|
||||
return target.request(MediaType.APPLICATION_JSON).get(ThreddsInfo.class);
|
||||
|
@ -112,7 +112,7 @@ public class ThreddsController {
|
|||
File temp=null;
|
||||
try{
|
||||
log.info("Locking remote path {} to processId {} ",operatingPath,processId);
|
||||
DataTransferClient cl=getDTClient(hostname);
|
||||
DataTransferClient cl=getDTClient(threddsEndpoints);
|
||||
|
||||
Destination dest=new Destination();
|
||||
dest.setCreateSubfolders(true);
|
||||
|
@ -143,7 +143,7 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
public void deleteThreddsFile(String location) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
log.info("Deleting file at {} ",urlString);
|
||||
try{
|
||||
getWebClient().target(urlString).request().delete();
|
||||
|
@ -163,7 +163,7 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
public String readThreddsFile(String location) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(location);
|
||||
log.info("Reading file at {} ",urlString);
|
||||
try{
|
||||
return getWebClient().target(urlString).request().get().readEntity(String.class);
|
||||
|
@ -176,8 +176,8 @@ public class ThreddsController {
|
|||
public void createEmptyFolder(String targetPath) throws InternalException {
|
||||
String toCleanPath=getPathFromStartingLocation(targetPath);
|
||||
try{
|
||||
log.info("Cleaning up {} on {} ",toCleanPath,hostname);
|
||||
DataTransferClient client=getDTClient(hostname);
|
||||
log.info("Cleaning up {} on {} ",toCleanPath,threddsEndpoints);
|
||||
DataTransferClient client=getDTClient(threddsEndpoints);
|
||||
File toTransfer=File.createTempFile("clean", ".dt_temp");
|
||||
toTransfer.createNewFile();
|
||||
Destination dest=new Destination();
|
||||
|
@ -188,7 +188,7 @@ public class ThreddsController {
|
|||
dest.setSubFolder(toCleanPath);
|
||||
dest.setDestinationFileName(toTransfer.getName());
|
||||
|
||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),hostname);
|
||||
log.info("Going to cleanup remote folder {} on {} ",dest.getSubFolder(),threddsEndpoints);
|
||||
client.localFile(toTransfer, dest);
|
||||
this.deleteThreddsFile(targetPath+"/"+toTransfer.getName());
|
||||
log.info("Done");
|
||||
|
@ -229,7 +229,7 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
public RemoteFileDescriptor getFileDescriptor(String path) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
log.info("Reading file at {} ",urlString);
|
||||
try{
|
||||
return getWebClient().target(urlString).queryParam("descriptor", true).request().get().readEntity(RemoteFileDescriptor.class);
|
||||
|
@ -239,7 +239,7 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
public InputStream getInputStream(String path) throws RemoteFileNotFoundException {
|
||||
String urlString="https://"+hostname+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
String urlString="https://"+threddsEndpoints.getHost()+"/"+Constants.THREDDS_DATA_TRANSFER_BASE_URL+getPathFromStartingLocation(path);
|
||||
log.info("Reading file at {} ",urlString);
|
||||
try{
|
||||
return getWebClient().target(urlString).request().get().readEntity(InputStream.class);
|
||||
|
@ -250,7 +250,7 @@ public class ThreddsController {
|
|||
|
||||
|
||||
public TransferResult transferFile(Destination dest,URL url,Set<PluginInvocation> invocations) throws InvalidSourceException, SourceNotSetException, FailedTransferException, InitializationException, InvalidDestinationException, DestinationNotSetException {
|
||||
DataTransferClient client=getDTClient(hostname);
|
||||
DataTransferClient client=getDTClient(threddsEndpoints);
|
||||
if(invocations!=null&&!invocations.isEmpty())
|
||||
return client.httpSource(url, dest,invocations);
|
||||
else return client.httpSource(url, dest);
|
||||
|
@ -268,7 +268,7 @@ public class ThreddsController {
|
|||
}
|
||||
|
||||
|
||||
private String getThreddsHost() throws InternalException{
|
||||
private URL getThreddsEndpoint() throws InternalException{
|
||||
|
||||
try{
|
||||
String sdiUrl="https://"+getSDIServiceHost()+"/"+Constants.SDI_CONFIG_PATH;
|
||||
|
@ -279,7 +279,7 @@ public class ThreddsController {
|
|||
throw new InternalException("Failed to contact SDI. Message "+respString);
|
||||
|
||||
DocumentContext sourceCtx=JsonPath.using(JSON_PATH_ALWAYS_LIST_CONFIG).parse(respString);
|
||||
return new URL(((List<String>) sourceCtx.read("$.threddsConfiguration..baseEndpoint")).get(0)).getHost();
|
||||
return new URL(((List<String>) sourceCtx.read("$.threddsConfiguration..baseEndpoint")).get(0));
|
||||
}catch(Throwable t) {
|
||||
log.error("Unable to read SDI configuration",t);
|
||||
throw new InternalException("Unable to gt configuration from SDI",t);
|
||||
|
@ -304,8 +304,8 @@ public class ThreddsController {
|
|||
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
|
||||
}
|
||||
|
||||
private static DataTransferClient getDTClient(String threddsHostName) throws UnreachableNodeException, ServiceNotFoundException {
|
||||
log.debug("Getting DT Client for {} ",threddsHostName);
|
||||
return DataTransferClient.getInstanceByEndpoint("https://"+threddsHostName);
|
||||
private static DataTransferClient getDTClient(URL threddsEndpoint) throws UnreachableNodeException, ServiceNotFoundException {
|
||||
log.debug("Getting DT Client for {} ",threddsEndpoint);
|
||||
return DataTransferClient.getInstanceByEndpoint(threddsEndpoint.toString());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
package org.gcube.usecases.ws.thredds.engine.impl.security;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
|
||||
import org.gcube.common.authorization.library.provider.ClientInfo;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.authorization.library.provider.UmaJWTProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
@ -17,17 +15,30 @@ public class Security {
|
|||
|
||||
|
||||
|
||||
public static User getCurrent() {
|
||||
ClientInfo client = AuthorizationProvider.instance.get().getClient();
|
||||
User toReturn = new User(client,UmaJWTProvider.instance.get(),ScopeProvider.instance.get());
|
||||
log.debug("Current User is {} ",toReturn);
|
||||
public static User getCurrent() throws SecurityException {
|
||||
String context=ScopeProvider.instance.get();
|
||||
if(context==null) throw new SecurityException("Cannot determine context");
|
||||
|
||||
log.debug("Context is {}, checking tokens..",context);
|
||||
|
||||
ClientInfo client = null;
|
||||
try{
|
||||
AuthorizationProvider.instance.get().getClient();
|
||||
}catch(Exception e) {
|
||||
log.warn("Unable to get client info ",e);
|
||||
}
|
||||
User toReturn = new User(client,UmaJWTProvider.instance.get(),SecurityTokenProvider.instance.get(),context);
|
||||
|
||||
log.info("Current User is {} ",toReturn);
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
public static void set(User toSet) {
|
||||
log.debug("Setting User {} ",toSet);
|
||||
UmaJWTProvider.instance.set(toSet.getToken());
|
||||
if(toSet.getUma_token()!=null)UmaJWTProvider.instance.set(toSet.getUma_token());
|
||||
if(toSet.getGcube_token()!=null)UmaJWTProvider.instance.set(toSet.getUma_token());
|
||||
if(ScopeProvider.instance.get()==null)ScopeProvider.instance.set(toSet.getContext());
|
||||
}
|
||||
|
||||
public static void checkOperator(SynchFolderConfiguration config) throws SecurityException{
|
||||
|
@ -48,8 +59,15 @@ public class Security {
|
|||
|
||||
public static String getContextFromgcubeToken(String token) throws ObjectNotFound, Exception {
|
||||
log.debug("Checking context of gcube-token {}...",token.substring(0,6));
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
String context=entry.getContext();
|
||||
return context;
|
||||
User caller=getCurrent();
|
||||
try {
|
||||
SecurityTokenProvider.instance.reset();
|
||||
UmaJWTProvider.instance.reset();
|
||||
SecurityTokenProvider.instance.set(token);
|
||||
return ScopeProvider.instance.get();
|
||||
}finally {
|
||||
log.debug("Resetting user "+caller);
|
||||
set(caller);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -9,7 +9,9 @@ import lombok.Getter;
|
|||
@Getter
|
||||
public class User{
|
||||
private ClientInfo user;
|
||||
private String token;
|
||||
private String uma_token;
|
||||
|
||||
private String gcube_token;
|
||||
|
||||
private String context;
|
||||
|
||||
|
@ -18,8 +20,12 @@ public class User{
|
|||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("User [user=");
|
||||
builder.append(user);
|
||||
builder.append(", token=");
|
||||
builder.append(token==null?token:"***");
|
||||
builder.append(", uma_token=");
|
||||
builder.append(uma_token==null?uma_token:"***");
|
||||
|
||||
builder.append(", gcube_token=");
|
||||
builder.append(gcube_token==null?gcube_token:"***");
|
||||
|
||||
builder.append(", context=");
|
||||
builder.append(context);
|
||||
builder.append("]");
|
||||
|
|
|
@ -2,6 +2,7 @@ package org.gcube.usecases.ws.thredds;
|
|||
|
||||
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
||||
|
||||
public class DTTests {
|
||||
|
@ -13,7 +14,7 @@ public class DTTests {
|
|||
StorageHubClient client=WorkspaceUtils.getClient();
|
||||
|
||||
String folderId=TestCommons.getByPath("/Workspace/Accounting").getId();
|
||||
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", TokenSetter.getCurrentToken(), "dummy",folderId), folderId);
|
||||
SyncEngine.get().setSynchronizedFolder(new SynchFolderConfiguration("another", "", Security.getCurrent().getContext(), "dummy",folderId), folderId);
|
||||
System.out.println("Done");
|
||||
|
||||
}
|
||||
|
|
|
@ -9,10 +9,10 @@ public class GetTrhreddsInfoTest {
|
|||
SyncEngine engine=SyncEngine.get();
|
||||
// ROOT
|
||||
|
||||
TokenSetter.setToken("***REMOVED***");
|
||||
TokenSetter.setGcubeToken("***REMOVED***");
|
||||
System.out.println(engine.getAvailableCatalogs());
|
||||
// MEI
|
||||
TokenSetter.setToken("54f577de-d259-407e-b30d-29bf9e7c0dee-843339462");
|
||||
TokenSetter.setGcubeToken("54f577de-d259-407e-b30d-29bf9e7c0dee-843339462");
|
||||
System.out.println(engine.getAvailableCatalogs());
|
||||
}
|
||||
|
||||
|
|
|
@ -46,11 +46,11 @@ public class TestCommons {
|
|||
|
||||
// folderName="WS-Tests";
|
||||
|
||||
configs.put("simple", new TestSet("Simple label ","/gcube/devsec/devVRE", "Test1","public/netcdf/simpleFolder","***REMOVED***","simple"));
|
||||
configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
|
||||
|
||||
configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
|
||||
|
||||
configs.put("simple", new TestSet("Simple label ","/gcube/devsec/devVRE", "Test1","public/netcdf/simpleFolder","3aa85bbf-d5f3-4df4-ad03-4f8f788eba3d-98187548","simple"));
|
||||
// configs.put("cmems", new TestSet("CMEMS","/gcube", "CMEMS","public/netcdf/CMEMS","***REMOVED***","cmems"));
|
||||
//
|
||||
// configs.put("default", new TestSet("Default Tests","/gcube","Thredds Catalog","public/netcdf","***REMOVED***","main"));
|
||||
//
|
||||
// configs.put("default", new TestSet("Default Tests","/gcube","WS-Tests","simpleCatalog","***REMOVED***","main"));
|
||||
//
|
||||
//
|
||||
|
|
|
@ -0,0 +1,30 @@
|
|||
package org.gcube.usecases.ws.thredds;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.Security;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.security.User;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class TokenSetter {
|
||||
|
||||
private static Properties props=null;
|
||||
|
||||
|
||||
public static synchronized void set(String scope){
|
||||
if(props==null) {
|
||||
props=new Properties();
|
||||
try {
|
||||
props.load(TokenSetter.class.getResourceAsStream("/tokens.properties"));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("YOU NEED TO SET TOKEN FILE IN CONFIGURATION");
|
||||
}
|
||||
}
|
||||
if(!props.containsKey(scope)) throw new RuntimeException("No token found for scope : "+scope);
|
||||
Security.set(new User(null, null, props.getProperty(scope), scope));
|
||||
}
|
||||
|
||||
}
|
|
@ -2,6 +2,7 @@ package org.gcube.usecases.ws.thredds;
|
|||
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessDescriptor;
|
||||
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
|
||||
|
@ -20,6 +21,7 @@ public class WorkspaceSynchronizationTest {
|
|||
|
||||
//TEST INFO...
|
||||
TestCommons.setScope();
|
||||
System.out.println(ScopeProvider.instance.get());
|
||||
FolderContainer folder=TestCommons.getTestFolder();
|
||||
|
||||
// FOLDER CONFIGURATION BEAN
|
||||
|
|
Loading…
Reference in New Issue