2021-07-14 17:15:03 +02:00
{
"ownerApp" : "Orchestrator",
"name" : "da_cache_to_shub",
"createBy" : "Marco Lettere",
"description": "Workflow for streaming da_cache files to storage hub",
"version" : 1,
"ownerEmail" : "marco.lettere@nubisware.com",
"inputParameters" : ["token", "descriptor"],
"tasks" : [
{
2021-07-16 10:46:36 +02:00
"name": "LAMBDA_TASK",
2021-09-13 15:00:27 +02:00
"taskReferenceName": "init",
2021-07-16 10:46:36 +02:00
"type": "LAMBDA",
"inputParameters": {
2021-07-16 11:21:55 +02:00
"downloads" : "${workflow.input.descriptor.downloads.*}",
2021-09-13 15:00:27 +02:00
"storagehub" : "{{ storagehub }}",
2021-09-17 09:31:20 +02:00
"workspace" : "{{ workspace }}",
2021-10-15 12:02:19 +02:00
"workspace_group" : "{{ workspace_group }}",
2021-07-16 10:46:36 +02:00
"auth" : "Bearer ${workflow.input.token}",
2021-10-13 13:10:15 +02:00
"parentfolder" : "Blue-Cloud Data Pool",
2021-09-17 15:12:32 +02:00
"folder" : "${workflow.input.descriptor.userOrderName}-${workflow.input.descriptor.orderNumber}",
2021-10-14 12:43:28 +02:00
"bcc" : ["prog@maris.nl","marco.lettere@nubisware.com"],
2021-10-15 18:04:30 +02:00
"max_download_size" : 16106127360,
2021-09-13 15:00:27 +02:00
"scriptExpression": "1 === 1"
2021-07-16 10:46:36 +02:00
}
},
{
2021-09-13 15:00:27 +02:00
"name" : "http",
"taskReferenceName" : "get_workspace",
"type" : "HTTP",
"inputParameters" : {
"http_request": {
"uri" : "${init.input.storagehub}/workspace",
"method" : "GET",
2021-09-13 15:36:02 +02:00
"headers" : { "Authorization" : "${init.input.auth}" }
2021-09-13 15:00:27 +02:00
}
}
2021-07-16 10:46:36 +02:00
},
{
2021-09-13 15:00:27 +02:00
"name": "LAMBDA_TASK",
"taskReferenceName": "init2",
"type": "LAMBDA",
"inputParameters": {
2021-09-13 15:38:16 +02:00
"workspace_root_folder_id" : "${get_workspace.output.response.body.item.id}",
"scriptExpression": "1 === 1"
2021-09-13 15:00:27 +02:00
}
},
{
"name" : "http",
"taskReferenceName" : "check_folder",
"type" : "HTTP",
"inputParameters" : {
"http_request": {
"uri" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/items/${init.input.parentfolder}",
"method" : "GET",
2021-09-13 15:36:02 +02:00
"headers" : { "Authorization" : "${init.input.auth}" }
2021-09-13 15:00:27 +02:00
}
}
},
2021-09-13 16:11:26 +02:00
{
"name": "decision",
"taskReferenceName": "check_user_existance",
"inputParameters": {
2021-09-13 16:20:09 +02:00
"folders": "${check_folder.output.response.body.itemlist}"
2021-09-13 16:11:26 +02:00
},
"type": "DECISION",
"caseExpression": "($.folders.length === 0 ? 'create' : 'exists')",
"decisionCases": {
"create": [
{
"name" : "pyrest",
"taskReferenceName" : "create_parent_folder",
"type" : "SIMPLE",
"inputParameters" : {
2021-09-13 16:14:57 +02:00
"url" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/create/FOLDER",
2021-09-13 16:11:26 +02:00
"method" : "POST",
"headers" : {
"Content-type": "application/x-www-form-urlencoded",
2021-09-13 16:22:26 +02:00
"Accept" : "text/plain",
"Authorization" : "${init.input.auth}"
2021-09-13 16:11:26 +02:00
},
"body" : {
"name" : "${init.input.parentfolder}",
2021-09-17 09:30:08 +02:00
"description" : "Root folder of all ${init.input.parentfolder}",
2021-09-13 16:11:26 +02:00
"hidden" : "false"
}
}
},
{
2021-09-13 16:38:23 +02:00
"type": "SET_VARIABLE",
2021-09-13 16:39:28 +02:00
"name": "store_parent_folderid_1",
2021-09-13 16:40:35 +02:00
"taskReferenceName": "store_parent_folderid_1",
2021-09-13 16:11:26 +02:00
"inputParameters": {
2021-09-13 16:56:02 +02:00
"parent_folder_id": "${create_parent_folder.output.body}"
2021-09-13 16:11:26 +02:00
}
}
],
"exists" : [
{
2021-09-13 16:38:23 +02:00
"type": "SET_VARIABLE",
2021-09-13 16:39:28 +02:00
"name": "store_parent_folderid_2",
2021-09-13 16:40:35 +02:00
"taskReferenceName": "store_parent_folderid_2",
2021-09-13 16:11:26 +02:00
"inputParameters": {
2021-09-13 16:38:23 +02:00
"parent_folder_id": "${check_folder.output.response.body.itemlist[0].id}"
2021-09-13 16:11:26 +02:00
}
}
]
}
},
2021-09-13 16:38:23 +02:00
{
"name" : "pyrest",
"taskReferenceName" : "create_destination_folder",
"type" : "SIMPLE",
"inputParameters" : {
"url" : "${init.input.storagehub}/workspace/items/${workflow.variables.parent_folder_id}/create/FOLDER",
"method" : "POST",
"headers" : {
"Content-type": "application/x-www-form-urlencoded",
"Accept" : "text/plain",
"Authorization" : "${init.input.auth}"
},
"body" : {
"name" : "${init.input.folder}",
"description" : "Target folder of Bluecloud data downloads",
"hidden" : "false"
}
}
},
{
"type": "SET_VARIABLE",
"name": "store_target_folderid",
"taskReferenceName": "store_target_folderid",
"inputParameters": {
2021-09-13 16:56:02 +02:00
"target_folder_id": "${create_destination_folder.output.body}"
2021-09-13 16:38:23 +02:00
}
},
2021-09-13 15:00:27 +02:00
{
"name": "LAMBDA_TASK",
"taskReferenceName": "build_transfer_tasks",
"type": "LAMBDA",
"inputParameters": {
2021-09-13 16:38:23 +02:00
"downloads" : "${workflow.input.descriptor.downloads.*}",
2021-09-15 11:25:28 +02:00
"upload" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/create/ARCHIVE",
2021-09-13 15:00:27 +02:00
"auth" : "Bearer ${workflow.input.token}",
2021-09-17 15:08:15 +02:00
"description" : "${workflow.input.descriptor.userOrderName}",
2021-10-15 18:04:30 +02:00
"max" : "${init.input.max_download_size}",
"scriptExpression": "inputs={};tasks=[];rejected=[];for(var i=0;i<$.downloads.length;i++) if($.downloads[i].size > $.max) rejected.push($.downloads[i]); else download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),rejected:Java.to(rejected,'java.util.Map[]'),inputs:inputs};"
2021-09-13 15:00:27 +02:00
}
2021-09-13 16:38:23 +02:00
},
{
"name" : "fork_dynamic",
"type" : "FORK_JOIN_DYNAMIC",
"taskReferenceName" : "parallel_downloads",
"inputParameters" : {
"tasks" : "${build_transfer_tasks.output.result.tasks}",
"inputs" : "${build_transfer_tasks.output.result.inputs}"
},
"dynamicForkTasksParam": "tasks",
"dynamicForkTasksInputParamName": "inputs"
},
{
"name" : "join",
"type" : "JOIN",
"taskReferenceName" : "join_parallel_downloads"
2021-09-16 16:59:35 +02:00
},
{
"name" : "fork_join",
"taskReferenceName" : "reporting",
"type": "FORK_JOIN",
"forkTasks" : [
[
{
"name": "LAMBDA_TASK",
"taskReferenceName": "build_json_report",
"type": "LAMBDA",
"inputParameters": {
"input" : "${workflow.input.descriptor}",
2021-09-16 17:14:59 +02:00
"output" : "${join_parallel_downloads.output}",
2021-10-15 18:04:30 +02:00
"rejected" : "${build_transfer_tasks.output.result.rejected}",
"scriptExpression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.descriptor.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};for(var r=0; r< $.rejected.length;r++) errors.push({url:$.rejected[r].url,number:$.rejected[r].number,error_number:9999,error_message:'Rejected because size is larger than allowed (16GB)'});report['errors']=Java.to(errors,'java.util.Map[]');return {report:report}"
2021-09-16 16:59:35 +02:00
}
2021-09-16 18:45:13 +02:00
},
2021-09-23 08:55:43 +02:00
{
"name" : "pyrest",
2021-09-23 08:59:05 +02:00
"taskReferenceName" : "bc_data_api_authorize",
2021-09-23 08:55:43 +02:00
"type" : "SIMPLE",
"inputParameters" : {
2021-09-23 08:59:05 +02:00
"url" : "${workflow.input.callback.authorize_url}",
2021-09-23 08:55:43 +02:00
"method" : "POST",
"headers" : {
2021-09-23 09:20:53 +02:00
"Content-Type": "application/json",
2021-09-23 08:55:43 +02:00
"Accept": "application/json"
},
"body" : {
"username": "${workflow.input.callback.username}",
"password": "${workflow.input.callback.password}"
}
}
},
2021-09-16 18:45:13 +02:00
{
2021-09-17 14:43:59 +02:00
"name" : "pyrest",
2021-09-16 18:45:13 +02:00
"taskReferenceName" : "send_json_report",
2021-09-17 14:43:59 +02:00
"type" : "SIMPLE",
2021-09-16 18:45:13 +02:00
"inputParameters" : {
2021-09-17 14:43:59 +02:00
"expect" : [200, 301],
"url" : "${workflow.input.callback.url}",
"method" : "POST",
"headers" : {
2021-09-22 12:15:47 +02:00
"Authorization" : "Bearer ${bc_data_api_authorize.output.body.token}",
2021-09-23 09:25:50 +02:00
"Accept" : "application/json",
2021-09-23 09:23:32 +02:00
"Content-Type": "application/json"
2021-09-17 14:43:59 +02:00
},
"body" : "${build_json_report.output.result.report}"
2021-09-16 18:45:13 +02:00
}
2021-09-17 14:46:16 +02:00
}
2021-09-16 16:59:35 +02:00
],
[
{
"name": "LAMBDA_TASK",
"taskReferenceName": "build_email_report",
"type": "LAMBDA",
"inputParameters": {
"input" : "${workflow.input.descriptor}",
2021-09-16 17:14:59 +02:00
"output" : "${join_parallel_downloads.output}",
2021-10-14 12:56:17 +02:00
"scriptExpression": "var k,report='<table style=\"border: solid 1px gray\"><thead style=\"background-color:rgba(0,0,0,.1)\"><tr><th>Download</th><th style=\"width:100%;text-align:left;\">Outcome</th><!--th>Link</th--></tr></thead><tbody>',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='<tr><td>'+(Number(index)+1)+'</td>';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='<td>'+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'</td>':report+='<td>OK</td>',report+='</tr>'}report+='</tbody></table>';return { report : report}"
2021-09-16 16:59:35 +02:00
}
2021-09-16 18:45:13 +02:00
},
2021-10-15 18:04:30 +02:00
{
"name": "LAMBDA_TASK",
"taskReferenceName": "build_rejected_email_report",
"type": "LAMBDA",
"inputParameters": {
"rejected" : "${build_transfer_tasks.output.result.rejected}",
"scriptExpression": "if($.rejected.length === 0) return {report: ''}; var k,report='<table style=\"border: solid 1px gray\"><thead style=\"background-color:rgba(0,0,0,.1)\"><tr><th>Download</th><th style=\"width:100%;text-align:left;\">Rejected</th></tr></thead><tbody>';for(var r=0; r< $.rejected.length;r++){report+='<tr><td>'+$.rejected[r].number+'</td><td>'+$.rejected[r].size+' exceeds max (16GB)</td></tr>'}report+='</tbody></table>';return { report : report}"
}
},
2021-09-16 18:45:13 +02:00
{
"name" : "pymail",
"taskReferenceName" : "send_email_report",
"type" : "SIMPLE",
"inputParameters" : {
2021-09-17 15:16:46 +02:00
"from" : "Blue-Cloud Gateway <bluecloud@d4science.org>",
2021-09-16 18:45:13 +02:00
"to" : "${workflow.input.callback.email}",
2021-10-13 17:26:49 +02:00
"bcc" : "${init.input.bcc}",
2021-10-14 13:43:39 +02:00
"html" : "<html><body style='font-family:ubuntu,Arial,sans-serif; color:#3a628b; display:flex; flex-direction: column;'><div><header style='background-color:#1aabe2;width:100%'><img src='https://ftp.d4science.org/apps/profiles/blue-cloud_vre_data_pool.jpg' style='margin-bottom:1em;width:100%'/></header><div style='width:100%'><p>Dear Blue-cloud service user,</p><p>your request related to order <b>{v2}</b>, number <b>{v3}</b>, has completed with the following outcome:</p><br/>{v4}<br/><p>You can access your data through your Blue-cloud workspace at <b><a href='{v5}'>{v6}</a></b>.</p><br/><br/><p style='font-size:small;color:gray'>WARNING / LEGAL TEXT: This message is intended only for the use of the individual or entity to which it is addressed and may contain information which is privileged, confidential, proprietary, or exempt from disclosure under applicable law. If you are not the intended recipient or the person responsible for delivering the message to the intended recipient, you are strictly prohibited from disclosing, distributing, copying, or in any way using this message. If you have received this communication in error, please notify the sender and destroy and delete any copies you may have received.</p></div></div></body></html>",
2021-09-17 09:00:12 +02:00
"subject" : "Outcome of Bluecloud DA cache download {v1}",
2021-09-16 18:45:13 +02:00
"extra_vars":{
2021-09-17 15:12:32 +02:00
"v1" : "${workflow.input.descriptor.requestId}",
2021-09-17 15:08:15 +02:00
"v2" : "${workflow.input.descriptor.userOrderName}",
2021-09-17 15:12:32 +02:00
"v3" : "${workflow.input.descriptor.orderNumber}",
2021-10-15 18:04:30 +02:00
"v4" : "${build_email_report.output.result.report}<br/>${build_rejected_email_report.output.result.report}",
2021-10-15 12:00:39 +02:00
"v5" : "${init.input.workspace}/group/${init.input.workspace_group}/workspace?itemid=${workflow.variables.target_folder_id}&operation=gotofolder",
2021-09-17 15:48:20 +02:00
"v6" : "${init.input.parentfolder}/${init.input.folder}"
2021-09-16 18:45:13 +02:00
}
}
2021-09-16 16:59:35 +02:00
}
]
]
},
{
"name" : "join",
"type" : "JOIN",
"taskReferenceName" : "join_reporting",
2021-09-17 08:44:31 +02:00
"joinOn" : ["send_json_report", "send_email_report"]
2021-07-14 17:15:03 +02:00
}
]
}