diff --git a/templates/da_cache_to_shub.json.j2 b/templates/da_cache_to_shub.json.j2 index 468258f..71ed97e 100644 --- a/templates/da_cache_to_shub.json.j2 +++ b/templates/da_cache_to_shub.json.j2 @@ -83,39 +83,80 @@ } }, { - "name": "LAMBDA_TASK", - "taskReferenceName": "extract_folder_id_1", - "type": "LAMBDA", + "type": "SET_VARIABLE", + "name": "store_parent_folderid", + "taskReferenceName": "store_parent_folderid", "inputParameters": { - "resp" : "${create_parent_folder.output.response}", - "scriptExpression": "return { parent_folder_id : $.resp}" + "parent_folder_id": "${create_parent_folder.output.response.body}" } } ], "exists" : [ { - "name": "LAMBDA_TASK", - "taskReferenceName": "extract_folder_id_2", - "type": "LAMBDA", + "type": "SET_VARIABLE", + "name": "store_parent_folderid", + "taskReferenceName": "store_parent_folderid", "inputParameters": { - "resp" : "${check_folder.output.response.body.itemlist[0]}", - "scriptExpression": "return { parent_folder_id : $.resp.id}" + "parent_folder_id": "${check_folder.output.response.body.itemlist[0].id}" } } ] } }, + { + "name" : "pyrest", + "taskReferenceName" : "create_destination_folder", + "type" : "SIMPLE", + "inputParameters" : { + "url" : "${init.input.storagehub}/workspace/items/${workflow.variables.parent_folder_id}/create/FOLDER", + "method" : "POST", + "headers" : { + "Content-type": "application/x-www-form-urlencoded", + "Accept" : "text/plain", + "Authorization" : "${init.input.auth}" + }, + "body" : { + "name" : "${init.input.folder}", + "description" : "Target folder of Bluecloud data downloads", + "hidden" : "false" + } + } + }, + { + "type": "SET_VARIABLE", + "name": "store_target_folderid", + "taskReferenceName": "store_target_folderid", + "inputParameters": { + "target_folder_id": "${create_destination_folder.output.response.body}" + } + }, { "name": "LAMBDA_TASK", "taskReferenceName": "build_transfer_tasks", "type": "LAMBDA", "inputParameters": { - "work" : "${workflow.input.descriptor.downloads.*}", - "upload" : "https://workspace-repository.dev.d4science.org/storagehub/workspace/items/68e30cd7-8728-429e-9a98-50039e309095/create/FILE", + "downloads" : "${workflow.input.descriptor.downloads.*}", + "upload" : "${init.input.stuoragehub}/workspace/items/${workflow.variables.target_folder_id}/create/FILE", "auth" : "Bearer ${workflow.input.token}", "description" : "${workflow.input.descriptor.order_name}", "scriptExpression": "inputs={};tasks=[];for(var i=0;i<$.downloads.length;i++)download=$.downloads[i],tasks.push({name:'pyrestbridge',type:'SIMPLE',taskReferenceName:'bridge_'+i}),inputs['bridge_'+i]={ datamap : { place: 'files', name : 'file'}, endpoint1 : { url: download.url}, endpoint2 : { url: $.upload, headers: { Authorization : $.auth}, files: { name : 'file' + download.number, description : $.description[0]}} };return {tasks:Java.to(tasks,'java.util.Map[]'),inputs:inputs};" } + }, + { + "name" : "fork_dynamic", + "type" : "FORK_JOIN_DYNAMIC", + "taskReferenceName" : "parallel_downloads", + "inputParameters" : { + "tasks" : "${build_transfer_tasks.output.result.tasks}", + "inputs" : "${build_transfer_tasks.output.result.inputs}" + }, + "dynamicForkTasksParam": "tasks", + "dynamicForkTasksInputParamName": "inputs" + }, + { + "name" : "join", + "type" : "JOIN", + "taskReferenceName" : "join_parallel_downloads" } ] }