{ "ownerApp" : "Orchestrator", "name" : "da_cache_to_shub", "createBy" : "Marco Lettere", "description": "Workflow for streaming da_cache files to storage hub", "version" : 1, "ownerEmail" : "marco.lettere@nubisware.com", "inputParameters" : ["token", "descriptor"], "tasks" : [ { "name": "INLINE_TASK", "taskReferenceName": "init", "type": "INLINE", "inputParameters": { "downloads" : "${workflow.input.descriptor.downloads.*}", "storagehub" : "{{ storagehub }}", "workspace" : "{{ workspace }}", "workspace_group" : "{{ workspace_group }}", "auth" : "Bearer ${workflow.input.token}", "parentfolder" : "Blue-Cloud Data Pool", "folder" : "${workflow.input.descriptor.userOrderName}-${workflow.input.descriptor.orderNumber}", "bcc" : ["prog@maris.nl","orchestrator@d4science.org"], "max_download_size" : 16106127360, "evaluatorType" : "javascript", "expression": "function f(){return { folder : $.folder.replace(/[<>&:/|?%\\[\\]\\\\]/g,' ').trim()}} f()" } }, { "name" : "pyrest", "taskReferenceName" : "get_workspace", "type" : "SIMPLE", "inputParameters" : { "url" : "${init.input.storagehub}/workspace", "method" : "GET", "headers" : { "Authorization" : "${init.input.auth}", "Accept" : "application/json" } }, "retryCount" : 5, "retryDelaySeconds": 10 }, { "name": "INLINE_TASK", "taskReferenceName": "init2", "type": "INLINE", "inputParameters": { "workspace_root_folder_id" : "${get_workspace.output.body.item.id}", "evaluatorType" : "javascript", "expression": "1 === 1" } }, { "name" : "http", "taskReferenceName" : "check_folder", "type" : "HTTP", "inputParameters" : { "http_request": { "uri" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/items/${init.input.parentfolder}", "method" : "GET", "readTimeOut" : 1000, "headers" : { "Authorization" : "${init.input.auth}" } } }, "retryCount" : 5, "retryDelaySeconds": 10 }, { "name": "decision", "taskReferenceName": "check_folder_existance", "inputParameters": { "folders": "${check_folder.output.response.body.itemlist}" }, "type": "SWITCH", "evaluatorType" : "javascript", "expression": "($.folders.length === 0 ? 'create' : 'exists')", "decisionCases": { "create": [ { "name" : "pyrest", "taskReferenceName" : "create_parent_folder", "type" : "SIMPLE", "inputParameters" : { "url" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/create/FOLDER", "method" : "POST", "headers" : { "Content-type": "application/x-www-form-urlencoded", "Accept" : "text/plain", "Authorization" : "${init.input.auth}" }, "body" : { "name" : "${init.input.parentfolder}", "description" : "Root folder of all ${init.input.parentfolder}", "hidden" : "false" } } }, { "type": "SET_VARIABLE", "name": "store_parent_folderid_1", "taskReferenceName": "store_parent_folderid_1", "inputParameters": { "parent_folder_id": "${create_parent_folder.output.body}" } } ], "exists" : [ { "type": "SET_VARIABLE", "name": "store_parent_folderid_2", "taskReferenceName": "store_parent_folderid_2", "inputParameters": { "parent_folder_id": "${check_folder.output.response.body.itemlist[0].id}" } } ] } }, { "name" : "pyrest", "taskReferenceName" : "check_destination_folder", "type" : "SIMPLE", "inputParameters" : { "url" : "${init.input.storagehub}/workspace/items/${workflow.variables.parent_folder_id}/items/${init.output.result.folder}", "expect" : [200, 404], "method" : "GET", "headers" : { "Authorization" : "${init.input.auth}", "Accept" : "application/json" } }, "retryCount" : 5, "retryDelaySeconds": 10 }, { "name": "decision", "taskReferenceName": "check_destination_folder_existance", "inputParameters": { "folders": "${check_destination_folder.output.body.itemlist}", "status": "${check_destination_folder.output.status}" }, "type": "SWITCH", "evaluatorType" : "javascript", "expression": "((($.folders && $.folders.length === 0) || $.status === 404) ? 'create' : 'exists')", "decisionCases": { "create": [ { "name" : "pyrest", "taskReferenceName" : "create_destination_folder", "type" : "SIMPLE", "inputParameters" : { "url" : "${init.input.storagehub}/workspace/items/${workflow.variables.parent_folder_id}/create/FOLDER", "method" : "POST", "headers" : { "Content-type": "application/x-www-form-urlencoded", "Accept" : "text/plain", "Authorization" : "${init.input.auth}" }, "body" : { "name" : "${init.output.result.folder}", "description" : "Target folder of Bluecloud data downloads", "hidden" : "false" } } }, { "type": "SET_VARIABLE", "name": "store_target_folderid", "taskReferenceName": "store_target_folderid", "inputParameters": { "target_folder_id": "${create_destination_folder.output.body}" } }, { "name" : "pyrest", "taskReferenceName" : "ensure_destination_folder", "type" : "SIMPLE", "retryCount" : 5, "retryDelaySeconds": 10, "inputParameters" : { "url" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/items", "method" : "GET", "headers" : { "Accept" : "application/json", "Authorization" : "${init.input.auth}" } } } ], "exists" : [ { "name": "terminate", "taskReferenceName": "duplicate_folder", "inputParameters": { "terminationStatus": "FAILED", "workflowOutput": "Order already processed." }, "type": "TERMINATE", "startDelay": 0, "optional": false } ] } }, { "name": "INLINE_TASK", "taskReferenceName": "build_transfer_tasks", "type": "INLINE", "inputParameters": { "downloads" : "${workflow.input.descriptor.downloads.*}", "upload" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/create/ARCHIVE", "auth" : "Bearer ${workflow.input.token}", "description" : "${workflow.input.descriptor.userOrderName}", "max" : "${init.input.max_download_size}", "evaluatorType" : "javascript", "expression": "inputs={};tasks=[];rejected=[];function f(){for(var i=0;i<$.downloads.length;i++) if($.downloads[i].size > $.max) rejected.push($.downloads[i]); else download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),rejected:Java.to(rejected,'java.util.Map[]'),inputs:inputs};} f()" } }, { "name" : "fork_dynamic", "type" : "FORK_JOIN_DYNAMIC", "taskReferenceName" : "parallel_downloads", "inputParameters" : { "tasks" : "${build_transfer_tasks.output.result.tasks}", "inputs" : "${build_transfer_tasks.output.result.inputs}" }, "dynamicForkTasksParam": "tasks", "dynamicForkTasksInputParamName": "inputs" }, { "name" : "join", "type" : "JOIN", "taskReferenceName" : "join_parallel_downloads" }, { "name" : "fork_join", "taskReferenceName" : "reporting", "type": "FORK_JOIN", "forkTasks" : [ [ { "name": "INLINE_TASK", "taskReferenceName": "build_json_report", "type": "INLINE", "inputParameters": { "input" : "${workflow.input.descriptor}", "output" : "${join_parallel_downloads.output}", "rejected" : "${build_transfer_tasks.output.result.rejected}", "evaluatorType" : "javascript", "expression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};for(var r=0; r< $.rejected.length;r++) errors.push({url:$.rejected[r].url,number:$.rejected[r].number,error_number:9999,error_message:'Rejected because size is larger than allowed (16GB)'});report['errors']=Java.to(errors,'java.util.Map[]');function f(){return {report:report}} f()" } }, { "name" : "pyrest", "taskReferenceName" : "bc_data_api_authorize", "type" : "SIMPLE", "inputParameters" : { "url" : "${workflow.input.callback.authorize_url}", "method" : "POST", "headers" : { "Content-Type": "application/json", "Accept": "application/json" }, "body" : { "username": "${workflow.input.callback.username}", "password": "${workflow.input.callback.password}" } } }, { "name" : "pyrest", "taskReferenceName" : "send_json_report", "type" : "SIMPLE", "inputParameters" : { "expect" : [200, 301], "url" : "${workflow.input.callback.url}", "method" : "POST", "headers" : { "Authorization" : "Bearer ${bc_data_api_authorize.output.body.token}", "Accept" : "application/json", "Content-Type": "application/json" }, "body" : "${build_json_report.output.result.report}" } } ], [ { "name": "INLINE_TASK", "taskReferenceName": "build_email_report", "type": "INLINE", "inputParameters": { "input" : "${workflow.input.descriptor}", "output" : "${join_parallel_downloads.output}", "evaluatorType" : "javascript", "expression": "var k,report='',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='':report+='',report+=''}report+='
DownloadOutcome
'+(Number(index)+1)+''+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'OK
'; function f(){return { report : report}} f()" } }, { "name": "INLINE_TASK", "taskReferenceName": "build_rejected_email_report", "type": "INLINE", "inputParameters": { "rejected" : "${build_transfer_tasks.output.result.rejected}", "evaluatorType" : "javascript", "expression": "function f(){if($.rejected.length === 0) return {report: ''}; var k,report='

REJECTED:

';for(var r=0; r< $.rejected.length;r++){report+=''}report+='
DownloadRejection reason
'+$.rejected[r].number+''+$.rejected[r].size+' bytes exceeds maximum allowed of 15Gb.
';return { report : report}} f()" } }, { "name" : "pymail", "taskReferenceName" : "send_email_report", "type" : "SIMPLE", "inputParameters" : { "from" : "Blue-Cloud Gateway ", "to" : "${workflow.input.callback.email}", "bcc" : "${init.input.bcc}", "html" : "

Dear Blue-cloud service user,

your request related to order {v2}, number {v3}, has completed with the following outcome:


{v4}

You can access your data through your Blue-cloud workspace at {v6}.



WARNING / LEGAL TEXT: This message is intended only for the use of the individual or entity to which it is addressed and may contain information which is privileged, confidential, proprietary, or exempt from disclosure under applicable law. If you are not the intended recipient or the person responsible for delivering the message to the intended recipient, you are strictly prohibited from disclosing, distributing, copying, or in any way using this message. If you have received this communication in error, please notify the sender and destroy and delete any copies you may have received.

", "subject" : "Outcome of Bluecloud DA cache download {v1}", "extra_vars":{ "v1" : "${workflow.input.descriptor.requestId}", "v2" : "${workflow.input.descriptor.userOrderName}", "v3" : "${workflow.input.descriptor.orderNumber}", "v4" : "${build_email_report.output.result.report}
${build_rejected_email_report.output.result.report}", "v5" : "${init.input.workspace}/group/${init.input.workspace_group}/workspace?itemid=${workflow.variables.target_folder_id}&operation=gotofolder", "v6" : "${init.input.parentfolder}/${init.output.result.folder}" } } } ] ] }, { "name" : "join", "type" : "JOIN", "taskReferenceName" : "join_reporting", "joinOn" : ["send_json_report", "send_email_report"] } ] }