323 lines
14 KiB
Java
323 lines
14 KiB
Java
package org.gcube.usecases.ws.thredds.engine.impl.threads;
|
|
|
|
import java.io.File;
|
|
import java.io.InputStream;
|
|
import java.nio.file.Files;
|
|
import java.util.Collections;
|
|
import java.util.HashMap;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
|
|
import org.gcube.common.storagehub.client.dsl.ContainerType;
|
|
import org.gcube.common.storagehub.client.dsl.FileContainer;
|
|
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
|
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
|
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
|
import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
|
|
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
|
|
import org.gcube.common.storagehub.model.items.FolderItem;
|
|
import org.gcube.common.storagehub.model.items.Item;
|
|
import org.gcube.data.transfer.library.TransferResult;
|
|
import org.gcube.data.transfer.model.Destination;
|
|
import org.gcube.data.transfer.model.DestinationClashPolicy;
|
|
import org.gcube.data.transfer.model.ExecutionReport;
|
|
import org.gcube.data.transfer.model.ExecutionReport.ExecutionReportFlag;
|
|
import org.gcube.data.transfer.model.PluginInvocation;
|
|
import org.gcube.data.transfer.model.RemoteFileDescriptor;
|
|
import org.gcube.data.transfer.model.plugins.thredds.DataSet;
|
|
import org.gcube.data.transfer.model.plugins.thredds.ThreddsInfo;
|
|
import org.gcube.spatial.data.sdi.interfaces.Metadata;
|
|
import org.gcube.spatial.data.sdi.model.metadata.MetadataPublishOptions;
|
|
import org.gcube.spatial.data.sdi.model.metadata.MetadataReport;
|
|
import org.gcube.spatial.data.sdi.model.metadata.TemplateInvocationBuilder;
|
|
import org.gcube.spatial.data.sdi.plugins.SDIAbstractPlugin;
|
|
import org.gcube.usecases.ws.thredds.Constants;
|
|
import org.gcube.usecases.ws.thredds.NetUtils;
|
|
import org.gcube.usecases.ws.thredds.engine.impl.ProcessStatus;
|
|
import org.gcube.usecases.ws.thredds.engine.impl.ThreddsController;
|
|
import org.gcube.usecases.ws.thredds.engine.impl.WorkspaceUtils;
|
|
import org.gcube.usecases.ws.thredds.faults.CancellationException;
|
|
import org.gcube.usecases.ws.thredds.faults.DataTransferPluginError;
|
|
import org.gcube.usecases.ws.thredds.faults.RemoteFileNotFoundException;
|
|
import org.gcube.usecases.ws.thredds.model.StepReport;
|
|
import org.gcube.usecases.ws.thredds.model.StepReport.OperationType;
|
|
import org.gcube.usecases.ws.thredds.model.StepReport.Status;
|
|
import org.gcube.usecases.ws.thredds.model.SynchFolderConfiguration;
|
|
import org.gcube.usecases.ws.thredds.model.SynchronizedElementInfo.SynchronizationStatus;
|
|
|
|
import lombok.Synchronized;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
@Slf4j
|
|
public class SynchronizationThread implements Runnable {
|
|
|
|
private SynchronizationRequest theRequest;
|
|
|
|
public SynchronizationThread(SynchronizationRequest theRequest) {
|
|
super();
|
|
this.theRequest = theRequest;
|
|
}
|
|
|
|
@Override
|
|
public void run() {
|
|
ProcessIdProvider.instance.set(theRequest.getProcess().getDescriptor().getProcessId());
|
|
log.debug("Executing Transfer request {} ",theRequest);
|
|
RequestLogger.get().log(theRequest);
|
|
String reportMessage="Never started";
|
|
String reportItemName="Still Unknown";
|
|
Status toSetStatus=Status.ERROR;
|
|
|
|
FolderItem parentFolderItem=theRequest.getLocation();
|
|
try {
|
|
StorageHubClient client=WorkspaceUtils.getClient();
|
|
FolderContainer parentFolder=client.open(parentFolderItem.getId()).asFolder();
|
|
checkCancelledProcess();
|
|
SynchFolderConfiguration synchConfig=WorkspaceUtils.loadConfiguration(parentFolder);
|
|
ThreddsController controller=new ThreddsController(synchConfig.getRemotePath(), theRequest.getProcess().getDescriptor().getOperator());
|
|
|
|
if(theRequest instanceof TransferToThreddsRequest) {
|
|
TransferToThreddsRequest request=(TransferToThreddsRequest) theRequest;
|
|
Item item=request.getToTransfer();
|
|
FileContainer itemContainer=client.open(item.getId()).asFile();
|
|
//look for metadata in same folder
|
|
String itemName=item.getName();
|
|
reportItemName=itemName;
|
|
String toLookMetadataName=itemName.substring(0, itemName.lastIndexOf("."))+".xml";
|
|
FileContainer metadataItem=getFileByName(client.open(item.getParentId()).asFolder(),false,toLookMetadataName);
|
|
|
|
// if not present, generate with sis/geotk
|
|
Destination toSetDestination=new Destination();
|
|
toSetDestination.setCreateSubfolders(true);
|
|
toSetDestination.setDestinationFileName(itemName);
|
|
toSetDestination.setOnExistingFileName(DestinationClashPolicy.REWRITE);
|
|
toSetDestination.setOnExistingSubFolder(DestinationClashPolicy.APPEND);
|
|
toSetDestination.setPersistenceId(synchConfig.getRemotePersistence());
|
|
|
|
//NB ITEM IS SUPPOSED TO HAVE REMOTE PATH
|
|
String fileLocation=request.getLocation().getMetadata().getMap().get(Constants.WorkspaceProperties.REMOTE_PATH)+"";
|
|
toSetDestination.setSubFolder(fileLocation);
|
|
|
|
|
|
checkCancelledProcess();
|
|
|
|
|
|
Set<PluginInvocation> invocations=null;
|
|
if(metadataItem==null) {
|
|
log.debug("Metadata not found, asking SIS/GEOTK for generation..");
|
|
invocations=Collections.singleton(new PluginInvocation(Constants.SIS_PLUGIN_ID));
|
|
}
|
|
log.info("Transferring to {} with invocations {} ",toSetDestination,invocations);
|
|
|
|
ThreddsInfo info=controller.getThreddsInfo();
|
|
|
|
DataSet dataset=info.getDataSetFromLocation(info.getLocalBasePath()+"/"+fileLocation);
|
|
|
|
// ThreddsCatalog catalog=controller.getCatalog();
|
|
|
|
checkCancelledProcess();
|
|
|
|
TransferResult result=controller.transferFile(toSetDestination, itemContainer.getPublicLink(), invocations);
|
|
|
|
|
|
Map<String,Object> toSetProperties=new HashMap<String,Object>();
|
|
|
|
|
|
String toSetMetadataUUID=null;
|
|
|
|
Boolean validateMetadata=synchConfig.getValidateMetadata();
|
|
|
|
|
|
checkCancelledProcess();
|
|
if(metadataItem==null) {
|
|
ExecutionReport report=result.getExecutionReports().get(Constants.SIS_PLUGIN_ID);
|
|
if(!report.getFlag().equals(ExecutionReportFlag.SUCCESS)) throw new DataTransferPluginError("Unable to Extract Metadata for "+itemName+" Message is "+report.getMessage());
|
|
else toSetMetadataUUID=report.getMessage();
|
|
|
|
}else {
|
|
MetadataReport metaReport=publishMetadata(metadataItem,info.getHostname(),itemName,dataset.getPath(),validateMetadata);
|
|
toSetMetadataUUID=metaReport.getPublishedUUID();
|
|
}
|
|
|
|
toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, controller.getFileDescriptor(itemName).getLastUpdate()+"");
|
|
toSetProperties.put(Constants.WorkspaceProperties.METADATA_UUID, toSetMetadataUUID);
|
|
|
|
toSetProperties.put(Constants.WorkspaceProperties.SYNCHRONIZATION_STATUS, SynchronizationStatus.UP_TO_DATE+"");
|
|
org.gcube.common.storagehub.model.Metadata meta=item.getMetadata();
|
|
meta.setMap(toSetProperties);
|
|
item.setMetadata(meta);
|
|
|
|
|
|
reportMessage="Successfully transferred and published.";
|
|
// End ws->th
|
|
} else if(theRequest instanceof TransferFromThreddsRequest) {
|
|
Map<String,Object> toSetProperties=new HashMap<String,Object>();
|
|
TransferFromThreddsRequest importRequest=(TransferFromThreddsRequest) theRequest;
|
|
String toImportName=null;
|
|
FileContainer targetItem=null;
|
|
if(importRequest.getTargetItem()==null) {
|
|
//Target Item will be created
|
|
toImportName=importRequest.getRemoteFilename();
|
|
reportMessage="Importing remote file..";
|
|
}else {
|
|
//Target Item already exists
|
|
toImportName=importRequest.getTargetItem().getName();
|
|
targetItem=client.open(importRequest.getTargetItem().getId()).asFile();
|
|
reportMessage="Updating file..";
|
|
}
|
|
|
|
reportItemName=toImportName;
|
|
|
|
RemoteFileDescriptor toImport=controller.getFileDescriptor(toImportName);
|
|
|
|
toSetProperties.put(Constants.WorkspaceProperties.LAST_UPDATE_TIME, toImport.getLastUpdate()+"");
|
|
InputStream source=null;
|
|
try {
|
|
source=controller.getInputStream(toImportName);
|
|
targetItem=parentFolder.uploadFile(source, toImportName, "Imported from Thredds");
|
|
// if(targetItem==null)
|
|
// targetItem=parentFolder.createExternalFileItem(toImportName, "Imported from Thredds", null, source);
|
|
// else {
|
|
// targetItem.updateItem(source);
|
|
// }
|
|
|
|
|
|
org.gcube.common.storagehub.model.Metadata meta=targetItem.get().getMetadata();
|
|
meta.setMap(toSetProperties);
|
|
targetItem.setMetadata(meta);
|
|
}finally {
|
|
if(source!=null)
|
|
source.close();
|
|
}
|
|
reportMessage="File successfully imported";
|
|
|
|
|
|
|
|
|
|
}else if(theRequest instanceof DeleteRemoteRequest) {
|
|
DeleteRemoteRequest deleteRequest=(DeleteRemoteRequest) theRequest;
|
|
reportItemName=deleteRequest.getToRemoveName();
|
|
|
|
log.debug("Going to delete {} from ",reportItemName,synchConfig.getRemotePath());
|
|
|
|
RemoteFileDescriptor desc=controller.getFileDescriptor(reportItemName);
|
|
if(desc.isDirectory()) {
|
|
log.debug("Remote {} is directory.. Cleaning it up, first.",reportItemName);
|
|
controller.createEmptyFolder(null);
|
|
}
|
|
controller.deleteThreddsFile(reportItemName);
|
|
reportMessage="Successfully removed";
|
|
}
|
|
log.info("Synchronization of {} successful.",reportItemName);
|
|
toSetStatus=Status.OK;
|
|
}catch(CancellationException e) {
|
|
log.debug("Process cancelled.. ",e);
|
|
reportMessage="CancelledProcess";
|
|
toSetStatus=Status.CANCELLED;
|
|
}catch(DataTransferPluginError e) {
|
|
log.debug("Unable to extract metadata ",e);
|
|
reportMessage="Unable to extract metadata : "+e.getMessage();
|
|
toSetStatus=Status.ERROR;
|
|
}catch(RemoteFileNotFoundException e) {
|
|
log.debug("Remote File not found ",e);
|
|
reportMessage="Remote File not found : "+e.getMessage();
|
|
toSetStatus=Status.ERROR;
|
|
}catch(StorageHubException e) {
|
|
log.debug("Internal generic exception ",e);
|
|
reportMessage="Internal error : "+e.getMessage();
|
|
toSetStatus=Status.ERROR;
|
|
}catch(Throwable t) {
|
|
log.debug("Internal generic exception ",t);
|
|
reportMessage="Unexpected exception : "+t.getMessage();
|
|
toSetStatus=Status.ERROR;
|
|
}finally {
|
|
updateParentProperty(parentFolderItem, toSetStatus);
|
|
submitReport(reportItemName,reportMessage,toSetStatus);
|
|
ProcessIdProvider.instance.reset();
|
|
}
|
|
}
|
|
|
|
|
|
@Synchronized
|
|
private static void updateParentProperty(FolderItem folderItem,StepReport.Status toSetStatus) {
|
|
boolean retry=false;
|
|
do {
|
|
try {
|
|
retry=false;
|
|
FolderContainer folder=WorkspaceUtils.getClient().open(folderItem.getId()).asFolder();
|
|
org.gcube.common.storagehub.model.Metadata folderMeta=folder.get().getMetadata();
|
|
Map<String,Object> props=folderMeta.getMap();
|
|
String currentValue=props.get(Constants.WorkspaceProperties.LAST_UPDATE_STATUS)+"";
|
|
if(currentValue==null||currentValue.isEmpty()||currentValue.equals("null"))
|
|
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
|
|
else {
|
|
StepReport.Status currentWSStatus=StepReport.Status.valueOf(currentValue);
|
|
if(currentWSStatus.equals(StepReport.Status.OK)&&!toSetStatus.equals(currentWSStatus))
|
|
props.put(Constants.WorkspaceProperties.LAST_UPDATE_STATUS, toSetStatus+"");
|
|
}
|
|
|
|
folderMeta.setMap(props);
|
|
folder.setMetadata(folderMeta);
|
|
}catch(ItemLockedException e) {
|
|
log.debug("Item locked retrying.. ",e);
|
|
retry=true;
|
|
}catch(Throwable t) {
|
|
log.warn("Unable to update folder status ",t);
|
|
}
|
|
}while(retry);
|
|
}
|
|
|
|
|
|
private void checkCancelledProcess() throws CancellationException{
|
|
if(theRequest.getProcess().getStatus().
|
|
getStatus().equals(ProcessStatus.Status.STOPPED))
|
|
throw new CancellationException("Process "+theRequest.getProcess().getDescriptor().getProcessId()+" has been cancelled");
|
|
}
|
|
|
|
|
|
private void submitReport(String elementName,String message,StepReport.Status status) {
|
|
StepReport report=new StepReport(elementName,message,status,OperationType.WS_TO_TH,System.currentTimeMillis());
|
|
|
|
if(theRequest instanceof TransferToThreddsRequest) report.setOperationType(OperationType.WS_TO_TH);
|
|
else if(theRequest instanceof TransferFromThreddsRequest) report.setOperationType(OperationType.TH_TO_WS);
|
|
else if(theRequest instanceof DeleteRemoteRequest) report.setOperationType(OperationType.DELETE_REMOTE);
|
|
else throw new RuntimeException("Unknown operation request "+theRequest);
|
|
theRequest.getProcess().onStep(report);
|
|
}
|
|
|
|
|
|
|
|
|
|
private static MetadataReport publishMetadata(FileContainer toPublish, String threddsHostname,String filename,String publicPath,Boolean validate) throws Exception{
|
|
File tempMetaFile=null;
|
|
try {
|
|
Metadata meta=SDIAbstractPlugin.metadata().build();
|
|
tempMetaFile=NetUtils.download(toPublish.getPublicLink());
|
|
|
|
log.debug("Publishing metadata {} ",filename);
|
|
|
|
MetadataPublishOptions opts=new MetadataPublishOptions(
|
|
new TemplateInvocationBuilder().threddsOnlineResources(threddsHostname, filename, publicPath).get());
|
|
opts.setGeonetworkCategory("Datasets");
|
|
opts.setValidate(validate);
|
|
return meta.pushMetadata(tempMetaFile, opts);
|
|
}catch(Throwable t) {
|
|
if(tempMetaFile!=null) Files.deleteIfExists(tempMetaFile.toPath());
|
|
throw new Exception("Something went wrong while publishing metadata for "+filename+". Cause : "+t.getMessage(),t);
|
|
}
|
|
}
|
|
|
|
|
|
private static final FileContainer getFileByName(FolderContainer toLookIntoFolder,boolean recursive,String toLookForName) throws StorageHubException {
|
|
log.debug(String.format("Looking for %1$s into %2$s [recursive %3$s ]",toLookForName,toLookIntoFolder.get().getPath()+" ID "+toLookIntoFolder.getId(),recursive));
|
|
for(ItemContainer<?> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
|
|
if(!item.getType().equals(ContainerType.FOLDER)&&item.get().getName().equals(toLookForName)) return (FileContainer) item;
|
|
|
|
if(recursive) {
|
|
for(ItemContainer<?> item : toLookIntoFolder.list().withMetadata().withAccounting().getContainers())
|
|
if(item.getType().equals(ContainerType.FOLDER)) return getFileByName((FolderContainer) item, recursive, toLookForName);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|