diff --git a/templates/da_cache_to_shub.json.j2 b/templates/da_cache_to_shub.json.j2 index e4ef02b..983e5ca 100644 --- a/templates/da_cache_to_shub.json.j2 +++ b/templates/da_cache_to_shub.json.j2 @@ -8,9 +8,9 @@ "inputParameters" : ["token", "descriptor"], "tasks" : [ { - "name": "LAMBDA_TASK", + "name": "INLINE_TASK", "taskReferenceName": "init", - "type": "LAMBDA", + "type": "INLINE", "inputParameters": { "downloads" : "${workflow.input.descriptor.downloads.*}", "storagehub" : "{{ storagehub }}", @@ -21,7 +21,8 @@ "folder" : "${workflow.input.descriptor.userOrderName}-${workflow.input.descriptor.orderNumber}", "bcc" : ["prog@maris.nl","orchestrator@d4science.org"], "max_download_size" : 16106127360, - "scriptExpression": "return { folder : $.folder.replace(/[<>&:/|?%\\[\\]\\\\]/g,' ').trim()}" + "evaluatorType" : "javascript", + "expression": "function f(){return { folder : $.folder.replace(/[<>&:/|?%\\[\\]\\\\]/g,' ').trim()}} f()" } }, { @@ -40,12 +41,13 @@ "retryDelaySeconds": 10 }, { - "name": "LAMBDA_TASK", + "name": "INLINE_TASK", "taskReferenceName": "init2", - "type": "LAMBDA", + "type": "INLINE", "inputParameters": { "workspace_root_folder_id" : "${get_workspace.output.body.item.id}", - "scriptExpression": "1 === 1" + "evaluatorType" : "javascript", + "expression": "1 === 1" } }, { @@ -68,8 +70,9 @@ "inputParameters": { "folders": "${check_folder.output.response.body.itemlist}" }, - "type": "DECISION", - "caseExpression": "($.folders.length === 0 ? 'create' : 'exists')", + "type": "SWITCH", + "evaluatorType" : "javascript", + "expression": "($.folders.length === 0 ? 'create' : 'exists')", "decisionCases": { "create": [ { @@ -135,8 +138,9 @@ "folders": "${check_destination_folder.output.body.itemlist}", "status": "${check_destination_folder.output.status}" }, - "type": "DECISION", - "caseExpression": "((($.folders && $.folders.length === 0) || $.status === 404) ? 'create' : 'exists')", + "type": "SWITCH", + "evaluatorType" : "javascript", + "expression": "((($.folders && $.folders.length === 0) || $.status === 404) ? 'create' : 'exists')", "decisionCases": { "create": [ { @@ -198,16 +202,17 @@ } }, { - "name": "LAMBDA_TASK", + "name": "INLINE_TASK", "taskReferenceName": "build_transfer_tasks", - "type": "LAMBDA", + "type": "INLINE", "inputParameters": { "downloads" : "${workflow.input.descriptor.downloads.*}", "upload" : "${init.input.storagehub}/workspace/items/${workflow.variables.target_folder_id}/create/ARCHIVE", "auth" : "Bearer ${workflow.input.token}", "description" : "${workflow.input.descriptor.userOrderName}", "max" : "${init.input.max_download_size}", - "scriptExpression": "inputs={};tasks=[];rejected=[];for(var i=0;i<$.downloads.length;i++) if($.downloads[i].size > $.max) rejected.push($.downloads[i]); else download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),rejected:Java.to(rejected,'java.util.Map[]'),inputs:inputs};" + "evaluatorType" : "javascript", + "expression": "inputs={};tasks=[];rejected=[];function f(){for(var i=0;i<$.downloads.length;i++) if($.downloads[i].size > $.max) rejected.push($.downloads[i]); else download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'downloads_'+i}),inputs['downloads_'+i]={ fail : false, datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { parentFolderName : 'download_' + download.number}} };return {tasks:Java.to(tasks,'java.util.Map[]'),rejected:Java.to(rejected,'java.util.Map[]'),inputs:inputs};} f()" } }, { @@ -233,14 +238,15 @@ "forkTasks" : [ [ { - "name": "LAMBDA_TASK", + "name": "INLINE_TASK", "taskReferenceName": "build_json_report", - "type": "LAMBDA", + "type": "INLINE", "inputParameters": { "input" : "${workflow.input.descriptor}", "output" : "${join_parallel_downloads.output}", "rejected" : "${build_transfer_tasks.output.result.rejected}", - "scriptExpression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};for(var r=0; r< $.rejected.length;r++) errors.push({url:$.rejected[r].url,number:$.rejected[r].number,error_number:9999,error_message:'Rejected because size is larger than allowed (16GB)'});report['errors']=Java.to(errors,'java.util.Map[]');return {report:report}" + "evaluatorType" : "javascript", + "expression": "var k,report={requestId:$.input.requestId,orderNumber:$.input.orderNumber,errors:[]},keys=[],errors=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1],o=$.output[keys[k]];o.endpoint1.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:1002,error_message:o.endpoint1.error}),o.endpoint2.error&&errors.push({url:$.input.downloads[index].url,number:index,error_number:9999,error_message:o.endpoint2.error})};for(var r=0; r< $.rejected.length;r++) errors.push({url:$.rejected[r].url,number:$.rejected[r].number,error_number:9999,error_message:'Rejected because size is larger than allowed (16GB)'});report['errors']=Java.to(errors,'java.util.Map[]');function f(){return {report:report}} f()" } }, { @@ -279,22 +285,24 @@ ], [ { - "name": "LAMBDA_TASK", + "name": "INLINE_TASK", "taskReferenceName": "build_email_report", - "type": "LAMBDA", + "type": "INLINE", "inputParameters": { "input" : "${workflow.input.descriptor}", "output" : "${join_parallel_downloads.output}", - "scriptExpression": "var k,report='',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='':report+='',report+=''}report+='
DownloadOutcome
'+(Number(index)+1)+''+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'OK
';return { report : report}" + "evaluatorType" : "javascript", + "expression": "var k,report='',keys=[];for(var k in $.output) keys.push(k);keys.sort();for(k in keys){var index=keys[k].split('_')[1];report+='';var o=$.output[keys[k]];o.endpoint1.error||o.endpoint2.error?report+='':report+='',report+=''}report+='
DownloadOutcome
'+(Number(index)+1)+''+(o.endpoint1.error||'')+(o.endpoint2.error||'')+'OK
'; function f(){return { report : report}} f()" } }, { - "name": "LAMBDA_TASK", + "name": "INLINE_TASK", "taskReferenceName": "build_rejected_email_report", - "type": "LAMBDA", + "type": "INLINE", "inputParameters": { "rejected" : "${build_transfer_tasks.output.result.rejected}", - "scriptExpression": "if($.rejected.length === 0) return {report: ''}; var k,report='

REJECTED:

';for(var r=0; r< $.rejected.length;r++){report+=''}report+='
DownloadRejection reason
'+$.rejected[r].number+''+$.rejected[r].size+' bytes exceeds maximum allowed of 15Gb.
';return { report : report}" + "evaluatorType" : "javascript", + "expression": "function f(){if($.rejected.length === 0) return {report: ''}; var k,report='

REJECTED:

';for(var r=0; r< $.rejected.length;r++){report+=''}report+='
DownloadRejection reason
'+$.rejected[r].number+''+$.rejected[r].size+' bytes exceeds maximum allowed of 15Gb.
';return { report : report}} f()" } }, {