You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
202 lines
7.3 KiB
Django/Jinja
202 lines
7.3 KiB
Django/Jinja
{
|
|
"ownerApp" : "Orchestrator",
|
|
"name" : "test_da_cache_to_shub",
|
|
"createBy" : "Marco Lettere",
|
|
"description": "TEST Workflow for streaming da_cache files to storage hub",
|
|
"version" : 1,
|
|
"ownerEmail" : "marco.lettere@nubisware.com",
|
|
"inputParameters" : ["token", "descriptor"],
|
|
"tasks" : [
|
|
{
|
|
"name": "LAMBDA_TASK",
|
|
"taskReferenceName": "init",
|
|
"type": "LAMBDA",
|
|
"inputParameters": {
|
|
"downloads" : "${workflow.input.descriptor.downloads.*}",
|
|
"storagehub" : "{{ storagehub }}",
|
|
"workspace" : "{{ workspace }}",
|
|
"auth" : "Bearer ${workflow.input.token}",
|
|
"parentfolder" : "Blue-Cloud Data Downloads",
|
|
"folder" : "${workflow.input.descriptor.userOrderName}-${workflow.input.descriptor.orderNumber}",
|
|
"scriptExpression": "1 === 1"
|
|
}
|
|
},
|
|
{
|
|
"name" : "http",
|
|
"taskReferenceName" : "get_workspace",
|
|
"type" : "HTTP",
|
|
"inputParameters" : {
|
|
"http_request": {
|
|
"uri" : "${init.input.storagehub}/workspace",
|
|
"method" : "GET",
|
|
"headers" : { "Authorization" : "${init.input.auth}" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "LAMBDA_TASK",
|
|
"taskReferenceName": "init2",
|
|
"type": "LAMBDA",
|
|
"inputParameters": {
|
|
"workspace_root_folder_id" : "${get_workspace.output.response.body.item.id}",
|
|
"scriptExpression": "1 === 1"
|
|
}
|
|
},
|
|
{
|
|
"name" : "http",
|
|
"taskReferenceName" : "check_folder",
|
|
"type" : "HTTP",
|
|
"inputParameters" : {
|
|
"http_request": {
|
|
"uri" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/items/${init.input.parentfolder}",
|
|
"method" : "GET",
|
|
"headers" : { "Authorization" : "${init.input.auth}" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "decision",
|
|
"taskReferenceName": "check_user_existance",
|
|
"inputParameters": {
|
|
"folders": "${check_folder.output.response.body.itemlist}"
|
|
},
|
|
"type": "DECISION",
|
|
"caseExpression": "($.folders.length === 0 ? 'create' : 'exists')",
|
|
"decisionCases": {
|
|
"create": [
|
|
{
|
|
"name" : "pyrest",
|
|
"taskReferenceName" : "create_parent_folder",
|
|
"type" : "SIMPLE",
|
|
"inputParameters" : {
|
|
"url" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/create/FOLDER",
|
|
"method" : "POST",
|
|
"headers" : {
|
|
"Content-type": "application/x-www-form-urlencoded",
|
|
"Accept" : "text/plain",
|
|
"Authorization" : "${init.input.auth}"
|
|
},
|
|
"body" : {
|
|
"name" : "${init.input.parentfolder}",
|
|
"description" : "Root folder of all ${init.input.parentfolder}",
|
|
"hidden" : "false"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "SET_VARIABLE",
|
|
"name": "store_parent_folderid_1",
|
|
"taskReferenceName": "store_parent_folderid_1",
|
|
"inputParameters": {
|
|
"parent_folder_id": "${create_parent_folder.output.body}"
|
|
}
|
|
}
|
|
],
|
|
"exists" : [
|
|
{
|
|
"type": "SET_VARIABLE",
|
|
"name": "store_parent_folderid_2",
|
|
"taskReferenceName": "store_parent_folderid_2",
|
|
"inputParameters": {
|
|
"parent_folder_id": "${check_folder.output.response.body.itemlist[0].id}"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name" : "pyrest",
|
|
"taskReferenceName" : "create_destination_folder",
|
|
"type" : "SIMPLE",
|
|
"inputParameters" : {
|
|
"url" : "${init.input.storagehub}/workspace/items/${workflow.variables.parent_folder_id}/create/FOLDER",
|
|
"method" : "POST",
|
|
"headers" : {
|
|
"Content-type": "application/x-www-form-urlencoded",
|
|
"Accept" : "text/plain",
|
|
"Authorization" : "${init.input.auth}"
|
|
},
|
|
"body" : {
|
|
"name" : "${init.input.folder}",
|
|
"description" : "Target folder of Bluecloud data downloads",
|
|
"hidden" : "false"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "SET_VARIABLE",
|
|
"name": "store_target_folderid",
|
|
"taskReferenceName": "store_target_folderid",
|
|
"inputParameters": {
|
|
"target_folder_id": "${create_destination_folder.output.body}"
|
|
}
|
|
},
|
|
{
|
|
"name": "LAMBDA_TASK",
|
|
"taskReferenceName": "build_transfer_tasks",
|
|
"type": "LAMBDA",
|
|
"inputParameters": {
|
|
"downloads" : "${workflow.input.descriptor.downloads.*}",
|
|
"upload" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/create/ARCHIVE",
|
|
"auth" : "Bearer ${workflow.input.token}",
|
|
"description" : "${workflow.input.descriptor.userOrderName}",
|
|
"scriptExpression": "inputs={};tasks=[];for(var i=0;i<$.downloads.length;i++)download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),inputs:inputs};"
|
|
}
|
|
},
|
|
{
|
|
"name" : "fork_dynamic",
|
|
"type" : "FORK_JOIN_DYNAMIC",
|
|
"taskReferenceName" : "parallel_downloads",
|
|
"inputParameters" : {
|
|
"tasks" : "${build_transfer_tasks.output.result.tasks}",
|
|
"inputs" : "${build_transfer_tasks.output.result.inputs}"
|
|
},
|
|
"dynamicForkTasksParam": "tasks",
|
|
"dynamicForkTasksInputParamName": "inputs"
|
|
},
|
|
{
|
|
"name" : "join",
|
|
"type" : "JOIN",
|
|
"taskReferenceName" : "join_parallel_downloads"
|
|
},
|
|
{
|
|
"name" : "fork_join",
|
|
"taskReferenceName" : "reporting",
|
|
"type": "FORK_JOIN",
|
|
"forkTasks" : [
|
|
[
|
|
{
|
|
"name": "LAMBDA_TASK",
|
|
"taskReferenceName": "build_json_report",
|
|
"type": "LAMBDA",
|
|
"inputParameters": {
|
|
"input" : "${workflow.input.descriptor}",
|
|
"output" : "${join_parallel_downloads.output}",
|
|
"scriptExpression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.descriptor.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};report['errors']=Java.to(errors,'java.util.Map[]');return {report:report}"
|
|
}
|
|
}
|
|
],
|
|
[
|
|
{
|
|
"name": "LAMBDA_TASK",
|
|
"taskReferenceName": "build_email_report",
|
|
"type": "LAMBDA",
|
|
"inputParameters": {
|
|
"input" : "${workflow.input.descriptor}",
|
|
"output" : "${join_parallel_downloads.output}",
|
|
"scriptExpression": "var k,report='<table><thead><tr><th>Download</th><th>Outcome</th><!--th>Link</th--></tr></thead><tbody>',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='<tr><td>'+(Number(index)+1)+'</td>';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='<td>'+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'</td>':report+='<td>OK</td>',report+='</tr>'}report+='</tbody></table>';return { report : report}"
|
|
}
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"name" : "join",
|
|
"type" : "JOIN",
|
|
"taskReferenceName" : "join_reporting",
|
|
"joinOn" : ["build_json_report", "build_email_report"]
|
|
}
|
|
]
|
|
}
|
|
|