Compare commits
23 Commits
pre-reject
...
master
| Author | SHA1 | Date |
|---|---|---|
dcore94
|
b204136c6a | |
|
dcore94
|
1e55c0ed8e | |
|
dcore94
|
5fc95848c7 | |
|
dcore94
|
c7af6b7bf4 | |
|
dcore94
|
b19bf38817 | |
|
dcore94
|
3aee9bfb01 | |
|
dcore94
|
4f5f2062a0 | |
|
dcore94
|
deb07b9fab | |
|
dcore94
|
c804bd46e0 | |
|
dcore94
|
f9c2fc8e2d | |
|
dcore94
|
1c7044cfe3 | |
|
dcore94
|
d6cb633f9a | |
|
dcore94
|
22267d9128 | |
|
dcore94
|
6b995f9185 | |
|
dcore94
|
937f021a8d | |
|
dcore94
|
6ba6fab746 | |
|
dcore94
|
a6dc075063 | |
|
dcore94
|
929aa78362 | |
|
dcore94
|
c69630d922 | |
|
dcore94
|
967011d996 | |
|
dcore94
|
0ee189e8e0 | |
|
dcore94
|
c173f3441c | |
|
dcore94
|
ef9de00325 |
|
|
@ -8,9 +8,9 @@
|
|||
"inputParameters" : ["token", "descriptor"],
|
||||
"tasks" : [
|
||||
{
|
||||
"name": "LAMBDA_TASK",
|
||||
"name": "INLINE_TASK",
|
||||
"taskReferenceName": "init",
|
||||
"type": "LAMBDA",
|
||||
"type": "INLINE",
|
||||
"inputParameters": {
|
||||
"downloads" : "${workflow.input.descriptor.downloads.*}",
|
||||
"storagehub" : "{{ storagehub }}",
|
||||
|
|
@ -19,29 +19,35 @@
|
|||
"auth" : "Bearer ${workflow.input.token}",
|
||||
"parentfolder" : "Blue-Cloud Data Pool",
|
||||
"folder" : "${workflow.input.descriptor.userOrderName}-${workflow.input.descriptor.orderNumber}",
|
||||
"bcc" : ["prog@maris.nl","marco.lettere@nubisware.com"],
|
||||
"scriptExpression": "1 === 1"
|
||||
"bcc" : ["prog@maris.nl","orchestrator@d4science.org"],
|
||||
"max_download_size" : 16106127360,
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "function f(){return { folder : $.folder.replace(/[<>&:/|?%\\[\\]\\\\]/g,' ').trim()}} f()"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name" : "http",
|
||||
"name" : "pyrest",
|
||||
"taskReferenceName" : "get_workspace",
|
||||
"type" : "HTTP",
|
||||
"type" : "SIMPLE",
|
||||
"inputParameters" : {
|
||||
"http_request": {
|
||||
"uri" : "${init.input.storagehub}/workspace",
|
||||
"url" : "${init.input.storagehub}/workspace",
|
||||
"method" : "GET",
|
||||
"headers" : { "Authorization" : "${init.input.auth}" }
|
||||
}
|
||||
"headers" : {
|
||||
"Authorization" : "${init.input.auth}",
|
||||
"Accept" : "application/json"
|
||||
}
|
||||
},
|
||||
"retryCount" : 5,
|
||||
"retryDelaySeconds": 10
|
||||
},
|
||||
{
|
||||
"name": "LAMBDA_TASK",
|
||||
"name": "INLINE_TASK",
|
||||
"taskReferenceName": "init2",
|
||||
"type": "LAMBDA",
|
||||
"type": "INLINE",
|
||||
"inputParameters": {
|
||||
"workspace_root_folder_id" : "${get_workspace.output.response.body.item.id}",
|
||||
"scriptExpression": "1 === 1"
|
||||
"workspace_root_folder_id" : "${get_workspace.output.body.item.id}",
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "1 === 1"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -52,18 +58,22 @@
|
|||
"http_request": {
|
||||
"uri" : "${init.input.storagehub}/workspace/items/${init2.input.workspace_root_folder_id}/items/${init.input.parentfolder}",
|
||||
"method" : "GET",
|
||||
"readTimeOut" : 1000,
|
||||
"headers" : { "Authorization" : "${init.input.auth}" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"retryCount" : 5,
|
||||
"retryDelaySeconds": 10
|
||||
},
|
||||
{
|
||||
"name": "decision",
|
||||
"taskReferenceName": "check_user_existance",
|
||||
"taskReferenceName": "check_folder_existance",
|
||||
"inputParameters": {
|
||||
"folders": "${check_folder.output.response.body.itemlist}"
|
||||
},
|
||||
"type": "DECISION",
|
||||
"caseExpression": "($.folders.length === 0 ? 'create' : 'exists')",
|
||||
"type": "SWITCH",
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "($.folders.length === 0 ? 'create' : 'exists')",
|
||||
"decisionCases": {
|
||||
"create": [
|
||||
{
|
||||
|
|
@ -106,6 +116,34 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name" : "pyrest",
|
||||
"taskReferenceName" : "check_destination_folder",
|
||||
"type" : "SIMPLE",
|
||||
"inputParameters" : {
|
||||
"url" : "${init.input.storagehub}/workspace/items/${workflow.variables.parent_folder_id}/items/${init.output.result.folder}",
|
||||
"expect" : [200, 404],
|
||||
"method" : "GET",
|
||||
"headers" : {
|
||||
"Authorization" : "${init.input.auth}",
|
||||
"Accept" : "application/json"
|
||||
}
|
||||
},
|
||||
"retryCount" : 5,
|
||||
"retryDelaySeconds": 10
|
||||
},
|
||||
{
|
||||
"name": "decision",
|
||||
"taskReferenceName": "check_destination_folder_existance",
|
||||
"inputParameters": {
|
||||
"folders": "${check_destination_folder.output.body.itemlist}",
|
||||
"status": "${check_destination_folder.output.status}"
|
||||
},
|
||||
"type": "SWITCH",
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "((($.folders && $.folders.length === 0) || $.status === 404) ? 'create' : 'exists')",
|
||||
"decisionCases": {
|
||||
"create": [
|
||||
{
|
||||
"name" : "pyrest",
|
||||
"taskReferenceName" : "create_destination_folder",
|
||||
|
|
@ -119,7 +157,7 @@
|
|||
"Authorization" : "${init.input.auth}"
|
||||
},
|
||||
"body" : {
|
||||
"name" : "${init.input.folder}",
|
||||
"name" : "${init.output.result.folder}",
|
||||
"description" : "Target folder of Bluecloud data downloads",
|
||||
"hidden" : "false"
|
||||
}
|
||||
|
|
@ -134,15 +172,48 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"name": "LAMBDA_TASK",
|
||||
"name" : "pyrest",
|
||||
"taskReferenceName" : "ensure_destination_folder",
|
||||
"type" : "SIMPLE",
|
||||
"retryCount" : 5,
|
||||
"retryDelaySeconds": 10,
|
||||
"inputParameters" : {
|
||||
"url" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/items",
|
||||
"method" : "GET",
|
||||
"headers" : {
|
||||
"Accept" : "application/json",
|
||||
"Authorization" : "${init.input.auth}"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"exists" : [
|
||||
{
|
||||
"name": "terminate",
|
||||
"taskReferenceName": "duplicate_folder",
|
||||
"inputParameters": {
|
||||
"terminationStatus": "FAILED",
|
||||
"workflowOutput": "Order already processed."
|
||||
},
|
||||
"type": "TERMINATE",
|
||||
"startDelay": 0,
|
||||
"optional": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "INLINE_TASK",
|
||||
"taskReferenceName": "build_transfer_tasks",
|
||||
"type": "LAMBDA",
|
||||
"type": "INLINE",
|
||||
"inputParameters": {
|
||||
"downloads" : "${workflow.input.descriptor.downloads.*}",
|
||||
"upload" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/create/ARCHIVE",
|
||||
"auth" : "Bearer ${workflow.input.token}",
|
||||
"description" : "${workflow.input.descriptor.userOrderName}",
|
||||
"scriptExpression": "inputs={};tasks=[];for(var i=0;i<$.downloads.length;i++)download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),inputs:inputs};"
|
||||
"max" : "${init.input.max_download_size}",
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "inputs={};tasks=[];rejected=[];function f(){for(var i=0;i<$.downloads.length;i++) if($.downloads[i].size > $.max) rejected.push($.downloads[i]); else download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),rejected:Java.to(rejected,'java.util.Map[]'),inputs:inputs};} f()"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -168,13 +239,15 @@
|
|||
"forkTasks" : [
|
||||
[
|
||||
{
|
||||
"name": "LAMBDA_TASK",
|
||||
"name": "INLINE_TASK",
|
||||
"taskReferenceName": "build_json_report",
|
||||
"type": "LAMBDA",
|
||||
"type": "INLINE",
|
||||
"inputParameters": {
|
||||
"input" : "${workflow.input.descriptor}",
|
||||
"output" : "${join_parallel_downloads.output}",
|
||||
"scriptExpression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.descriptor.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};report['errors']=Java.to(errors,'java.util.Map[]');return {report:report}"
|
||||
"rejected" : "${build_transfer_tasks.output.result.rejected}",
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};for(var r=0; r< $.rejected.length;r++) errors.push({url:$.rejected[r].url,number:$.rejected[r].number,error_number:9999,error_message:'Rejected because size is larger than allowed (16GB)'});report['errors']=Java.to(errors,'java.util.Map[]');function f(){return {report:report}} f()"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -213,13 +286,24 @@
|
|||
],
|
||||
[
|
||||
{
|
||||
"name": "LAMBDA_TASK",
|
||||
"name": "INLINE_TASK",
|
||||
"taskReferenceName": "build_email_report",
|
||||
"type": "LAMBDA",
|
||||
"type": "INLINE",
|
||||
"inputParameters": {
|
||||
"input" : "${workflow.input.descriptor}",
|
||||
"output" : "${join_parallel_downloads.output}",
|
||||
"scriptExpression": "var k,report='<table style=\"border: solid 1px gray\"><thead style=\"background-color:rgba(0,0,0,.1)\"><tr><th>Download</th><th style=\"width:100%;text-align:left;\">Outcome</th><!--th>Link</th--></tr></thead><tbody>',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='<tr><td>'+(Number(index)+1)+'</td>';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='<td>'+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'</td>':report+='<td>OK</td>',report+='</tr>'}report+='</tbody></table>';return { report : report}"
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "var k,report='<table style=\"border: solid 1px gray\"><thead style=\"background-color:rgba(0,0,0,.1)\"><tr><th>Download</th><th style=\"width:100%;text-align:left;\">Outcome</th><!--th>Link</th--></tr></thead><tbody>',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='<tr><td>'+(Number(index)+1)+'</td>';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='<td>'+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'</td>':report+='<td>OK</td>',report+='</tr>'}report+='</tbody></table>'; function f(){return { report : report}} f()"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "INLINE_TASK",
|
||||
"taskReferenceName": "build_rejected_email_report",
|
||||
"type": "INLINE",
|
||||
"inputParameters": {
|
||||
"rejected" : "${build_transfer_tasks.output.result.rejected}",
|
||||
"evaluatorType" : "javascript",
|
||||
"expression": "function f(){if($.rejected.length === 0) return {report: ''}; var k,report='<p>REJECTED:</p><table style=\"border: solid 1px gray\"><thead style=\"background-color:rgba(0,0,0,.1)\"><tr><th>Download</th><th style=\"width:100%;text-align:left;\">Rejection reason</th></tr></thead><tbody>';for(var r=0; r< $.rejected.length;r++){report+='<tr><td>'+$.rejected[r].number+'</td><td>'+$.rejected[r].size+' bytes exceeds maximum allowed of 15Gb.</td></tr>'}report+='</tbody></table>';return { report : report}} f()"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -236,9 +320,9 @@
|
|||
"v1" : "${workflow.input.descriptor.requestId}",
|
||||
"v2" : "${workflow.input.descriptor.userOrderName}",
|
||||
"v3" : "${workflow.input.descriptor.orderNumber}",
|
||||
"v4" : "${build_email_report.output.result.report}",
|
||||
"v4" : "${build_email_report.output.result.report}<br/>${build_rejected_email_report.output.result.report}",
|
||||
"v5" : "${init.input.workspace}/group/${init.input.workspace_group}/workspace?itemid=${workflow.variables.target_folder_id}&operation=gotofolder",
|
||||
"v6" : "${init.input.parentfolder}/${init.input.folder}"
|
||||
"v6" : "${init.input.parentfolder}/${init.output.result.folder}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue