You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
ansible-role-conductor-work.../templates/da_cache_to_shub.json.j2

254 lines
10 KiB
Django/Jinja

{
"ownerApp" : "Orchestrator",
"name" : "da_cache_to_shub",
"createBy" : "Marco Lettere",
"description": "Workflow for streaming da_cache files to storage hub",
"version" : 1,
"ownerEmail" : "marco.lettere@nubisware.com",
"inputParameters" : ["token", "descriptor"],
"tasks" : [
{
"name": "LAMBDA_TASK",
"taskReferenceName": "init",
"type": "LAMBDA",
"inputParameters": {
"downloads" : "${workflow.input.descriptor.downloads.*}",
"storagehub" : "{{ storagehub }}",
"workspace" : "{{ workspace }}",
"auth" : "Bearer ${workflow.input.token}",
"parentfolder" : "Blue-Cloud Data Pool",
"folder" : "${workflow.input.descriptor.userOrderName}-${workflow.input.descriptor.orderNumber}",
"scriptExpression": "1 === 1"
}
},
{
"name" : "http",
"taskReferenceName" : "get_workspace",
"type" : "HTTP",
"inputParameters" : {
"http_request": {
"uri" : "${init.input.storagehub}/workspace",
"method" : "GET",
"headers" : { "Authorization" : "${init.input.auth}" }
}
}
},
{
"name": "LAMBDA_TASK",
"taskReferenceName": "init2",
"type": "LAMBDA",
"inputParameters": {
"workspace_root_folder_id" : "${get_workspace.output.response.body.item.id}",
"scriptExpression": "1 === 1"
}
},
{
"name" : "http",
"taskReferenceName" : "check_folder",
"type" : "HTTP",
"inputParameters" : {
"http_request": {
"uri" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/items/${init.input.parentfolder}",
"method" : "GET",
"headers" : { "Authorization" : "${init.input.auth}" }
}
}
},
{
"name": "decision",
"taskReferenceName": "check_user_existance",
"inputParameters": {
"folders": "${check_folder.output.response.body.itemlist}"
},
"type": "DECISION",
"caseExpression": "($.folders.length === 0 ? 'create' : 'exists')",
"decisionCases": {
"create": [
{
"name" : "pyrest",
"taskReferenceName" : "create_parent_folder",
"type" : "SIMPLE",
"inputParameters" : {
"url" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/create/FOLDER",
"method" : "POST",
"headers" : {
"Content-type": "application/x-www-form-urlencoded",
"Accept" : "text/plain",
"Authorization" : "${init.input.auth}"
},
"body" : {
"name" : "${init.input.parentfolder}",
"description" : "Root folder of all ${init.input.parentfolder}",
"hidden" : "false"
}
}
},
{
"type": "SET_VARIABLE",
"name": "store_parent_folderid_1",
"taskReferenceName": "store_parent_folderid_1",
"inputParameters": {
"parent_folder_id": "${create_parent_folder.output.body}"
}
}
],
"exists" : [
{
"type": "SET_VARIABLE",
"name": "store_parent_folderid_2",
"taskReferenceName": "store_parent_folderid_2",
"inputParameters": {
"parent_folder_id": "${check_folder.output.response.body.itemlist[0].id}"
}
}
]
}
},
{
"name" : "pyrest",
"taskReferenceName" : "create_destination_folder",
"type" : "SIMPLE",
"inputParameters" : {
"url" : "${init.input.storagehub}/workspace/items/${workflow.variables.parent_folder_id}/create/FOLDER",
"method" : "POST",
"headers" : {
"Content-type": "application/x-www-form-urlencoded",
"Accept" : "text/plain",
"Authorization" : "${init.input.auth}"
},
"body" : {
"name" : "${init.input.folder}",
"description" : "Target folder of Bluecloud data downloads",
"hidden" : "false"
}
}
},
{
"type": "SET_VARIABLE",
"name": "store_target_folderid",
"taskReferenceName": "store_target_folderid",
"inputParameters": {
"target_folder_id": "${create_destination_folder.output.body}"
}
},
{
"name": "LAMBDA_TASK",
"taskReferenceName": "build_transfer_tasks",
"type": "LAMBDA",
"inputParameters": {
"downloads" : "${workflow.input.descriptor.downloads.*}",
"upload" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/create/ARCHIVE",
"auth" : "Bearer ${workflow.input.token}",
"description" : "${workflow.input.descriptor.userOrderName}",
"scriptExpression": "inputs={};tasks=[];for(var i=0;i<$.downloads.length;i++)download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),inputs:inputs};"
}
},
{
"name" : "fork_dynamic",
"type" : "FORK_JOIN_DYNAMIC",
"taskReferenceName" : "parallel_downloads",
"inputParameters" : {
"tasks" : "${build_transfer_tasks.output.result.tasks}",
"inputs" : "${build_transfer_tasks.output.result.inputs}"
},
"dynamicForkTasksParam": "tasks",
"dynamicForkTasksInputParamName": "inputs"
},
{
"name" : "join",
"type" : "JOIN",
"taskReferenceName" : "join_parallel_downloads"
},
{
"name" : "fork_join",
"taskReferenceName" : "reporting",
"type": "FORK_JOIN",
"forkTasks" : [
[
{
"name": "LAMBDA_TASK",
"taskReferenceName": "build_json_report",
"type": "LAMBDA",
"inputParameters": {
"input" : "${workflow.input.descriptor}",
"output" : "${join_parallel_downloads.output}",
"scriptExpression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.descriptor.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};report['errors']=Java.to(errors,'java.util.Map[]');return {report:report}"
}
},
{
"name" : "pyrest",
"taskReferenceName" : "bc_data_api_authorize",
"type" : "SIMPLE",
"inputParameters" : {
"url" : "${workflow.input.callback.authorize_url}",
"method" : "POST",
"headers" : {
"Content-Type": "application/json",
"Accept": "application/json"
},
"body" : {
"username": "${workflow.input.callback.username}",
"password": "${workflow.input.callback.password}"
}
}
},
{
"name" : "pyrest",
"taskReferenceName" : "send_json_report",
"type" : "SIMPLE",
"inputParameters" : {
"expect" : [200, 301],
"url" : "${workflow.input.callback.url}",
"method" : "POST",
"headers" : {
"Authorization" : "Bearer ${bc_data_api_authorize.output.body.token}",
"Accept" : "application/json",
"Content-Type": "application/json"
},
"body" : "${build_json_report.output.result.report}"
}
}
],
[
{
"name": "LAMBDA_TASK",
"taskReferenceName": "build_email_report",
"type": "LAMBDA",
"inputParameters": {
"input" : "${workflow.input.descriptor}",
"output" : "${join_parallel_downloads.output}",
"scriptExpression": "var k,report='<table><thead><tr><th>Download</th><th>Outcome</th><!--th>Link</th--></tr></thead><tbody>',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='<tr><td>'+(Number(index)+1)+'</td>';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='<td>'+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'</td>':report+='<td>OK</td>',report+='</tr>'}report+='</tbody></table>';return { report : report}"
}
},
{
"name" : "pymail",
"taskReferenceName" : "send_email_report",
"type" : "SIMPLE",
"inputParameters" : {
"from" : "Blue-Cloud Gateway <bluecloud@d4science.org>",
"to" : "${workflow.input.callback.email}",
"html" : "<html><body><p>Dear user,</p> <p>your request with id <b>{v1}</b> related to order <b>{v2}</b> number <b>{v3}</b> has completed with the following outcome:</p><br/> {v4} <br/><p>You can access your data through your Blue-cloud workspace at <b><a href='{v5}'>{v6}</a></b>.</p><br/><br/><p style='font-size:small;color:gray'>WARNING / LEGAL TEXT: This message is intended only for the use of the individual or entity to which it is addressed and may contain information which is privileged, confidential, proprietary, or exempt from disclosure under applicable law. If you are not the intended recipient or the person responsible for delivering the message to the intended recipient, you are strictly prohibited from disclosing, distributing, copying, or in any way using this message. If you have received this communication in error, please notify the sender and destroy and delete any copies you may have received.</p></body></html>",
"subject" : "Outcome of Bluecloud DA cache download {v1}",
"extra_vars":{
"v1" : "${workflow.input.descriptor.requestId}",
"v2" : "${workflow.input.descriptor.userOrderName}",
"v3" : "${workflow.input.descriptor.orderNumber}",
"v4" : "${build_email_report.output.result.report}",
"v5" : "${init.input.workspace}/group/next/workspace?itemid=${workflow.variables.target_folder_id}&operation=gotofolder",
"v6" : "${init.input.parentfolder}/${init.input.folder}"
}
}
}
]
]
},
{
"name" : "join",
"type" : "JOIN",
"taskReferenceName" : "join_reporting",
"joinOn" : ["send_json_report", "send_email_report"]
}
]
}