1251 lines
96 KiB
XML
1251 lines
96 KiB
XML
<RESOURCE_PROFILE>
|
|
<HEADER>
|
|
<RESOURCE_IDENTIFIER value="4801c33c-66ca-4ab6-af64-aa812194ec61_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
|
|
<RESOURCE_TYPE value="WorkflowDSResourceType"/>
|
|
<RESOURCE_KIND value="WorkflowDSResources"/>
|
|
<RESOURCE_URI value=""/>
|
|
<DATE_OF_CREATION value="2023-06-07T08:37:57+00:00"/>
|
|
</HEADER>
|
|
<BODY>
|
|
<WORKFLOW_NAME>Graph construction and processing</WORKFLOW_NAME>
|
|
<WORKFLOW_TYPE>Data Provision</WORKFLOW_TYPE>
|
|
<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
|
|
<CONFIGURATION start="manual">
|
|
<NODE isStart="true" name="setParameters" type="SetEnvJsonParameters">
|
|
<DESCRIPTION>
|
|
- unresolvedEntityPath: path of unresolved entities
|
|
- nsPrefixBlacklist_PROD: blacklist of funder nsPrefixes from the prod aggregator
|
|
- idMappingPath: path of the map defining the relations id mappings
|
|
- dedupConfig: dedup orchestrator name
|
|
- actionSetIdsRawGraph: ActionSet ids to promote in the RAW graph
|
|
- isLookUpUrl: IS lookup service address
|
|
- iterations: number of iterations performed by the affiliation through semantic relation propagation algo
|
|
</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="user" name="json" required="true" type="string">
|
|
{
|
|
'unresolvedEntityPath' : '/data/unresolved_PROD/content',
|
|
'nsPrefixBlacklist' : 'conicytf____,dfgf________,gsrt________,innoviris___,miur________,rif_________,rsf_________,sgov________,sfrs________',
|
|
'idMappingPath' : '/data/maps/funder_remapping/',
|
|
'dedupConfig' : 'dedup-result-decisiontree-v3',
|
|
'actionSetIdsRawGraph' : 'doiboost,orcidworks-no-doi,ror',
|
|
'actionSetIdsIISGraph' : 'iis-researchinitiative,iis-document-citations,iis-document-citations-relations,iis-document-affiliation,iis-document-classes,iis-document-similarities,iis-referenced-datasets-main,iis-referenced-projects-main,iis-referenced-projects-backup,iis-referenceextraction-pdb,document_software_url,iis-extracted-metadata,iis-communities,iis-referenced-patents,iis-covid-19,opencitations,h2020classification,iis-entities-software,iis-entities-patent',
|
|
'actionSetIdsDedup' : 'bipfinder-scores,usage_counts',
|
|
'isLookUpUrl' : 'http://services.openaire.eu:8280/is/services/isLookUp?wsdl',
|
|
'iterations' : '1',
|
|
|
|
'reuseDBClaims' : 'false',
|
|
'reuseODFClaims' : 'false',
|
|
'reuseOAFClaims' : 'false',
|
|
'reuseODFhdfs' : 'false',
|
|
'reuseOAFhdfs' : 'false',
|
|
'reuseDB' : 'false',
|
|
'reuseDBOpenorgs' : 'false',
|
|
'reuseODF' : 'false',
|
|
'reuseOAF' : 'false',
|
|
'shouldPatchRelations' : 'true',
|
|
'contentPath' : '/tmp/prod_aggregator',
|
|
|
|
'aggregatorGraphPath' : '/tmp/prod_provision/graph/00_graph_aggregator',
|
|
'rawGraphPath' : '/tmp/prod_provision/graph/01_graph_raw',
|
|
'groupedGraphPath' : '/tmp/prod_provision/graph/02_graph_grouped',
|
|
'cleanedFirstGraphPath' : '/tmp/prod_provision/graph/03_graph_cleaned',
|
|
'resolvedGraphPath' : '/tmp/prod_provision/graph/04_graph_resolved',
|
|
'inferredGraphPath' : '/tmp/prod_provision/graph/05_graph_inferred',
|
|
'dedupGraphPath' : '/tmp/prod_provision/graph/06_graph_dedup',
|
|
'consistentGraphPath' : '/tmp/prod_provision/graph/07_graph_consistent',
|
|
'dedupEnrichedGraphPath' : '/tmp/prod_provision/graph/08_graph_dedup_enriched',
|
|
'orcidGraphPath' : '/tmp/prod_provision/graph/09_graph_orcid',
|
|
'bulkTaggingGraphPath' : '/tmp/prod_provision/graph/10_graph_bulktagging',
|
|
'affiliationGraphPath' : '/tmp/prod_provision/graph/11_graph_affiliation',
|
|
'affiliationSemRelGraphPath' : '/tmp/prod_provision/graph/12_graph_affiliation_semantic_relation',
|
|
'communityOrganizationGraphPath' : '/tmp/prod_provision/graph/13_graph_community_organization',
|
|
'fundingGraphPath' : '/tmp/prod_provision/graph/14_graph_funding',
|
|
'communitySemRelGraphPath' : '/tmp/prod_provision/graph/15_graph_community_semantic_relation',
|
|
'countryGraphPath' : '/tmp/prod_provision/graph/16_graph_country',
|
|
'cleanedGraphPath' : '/tmp/prod_provision/graph/17_graph_cleaned',
|
|
'blacklistedGraphPath' : '/tmp/prod_provision/graph/18_graph_blacklisted'
|
|
}
|
|
</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="resumeFrom"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE isStart="true" name="setResumeFromHBM" type="SetEnvParameter">
|
|
<DESCRIPTION>the step from which the hostedByMap step should be resumed. If the choice is DownloadBoth both the unibi gold and doaj files are downloaded anew. If the option id DownloadDoaj or DownloadGold only the selected file is downloaded. If the option id ProduceHBM a new hosted by map is produced. If the option is PrepareInfo the old HBM is used on the results and datasources. </DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="parameterName" required="true" type="string">HBMresumeFrom</PARAM>
|
|
<PARAM function="validValues(['DownloadBoth', 'DownloadGold', 'DownloadDoaj', 'ProduceHBM','PrepareInfo'])" managedBy="user" name="parameterValue" required="false" type="string">PrepareInfo</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="resumeFrom"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE isStart="true" name="setBulkTaggingPathMap" type="SetEnvParameter">
|
|
<DESCRIPTION>Set the map of paths for the Bulk Tagging</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="parameterName" required="true" type="string">bulkTaggingPathMap</PARAM>
|
|
<PARAM managedBy="system" name="parameterValue" required="true" type="string">{"author" : "$['author'][*]['fullname']", "title" : "$['title'][*]['value']","orcid":"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid')]['value']", "contributor" : "$['contributor'][*]['value']", "description" : "$['description'][*]['value']"}</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="resumeFrom"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE isStart="true" name="setPropagationOrganizationCommunityMap" type="SetEnvParameter">
|
|
<DESCRIPTION>Set the map of associations organization, community list for the propagation of
|
|
community to result through organization</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="parameterName" required="true" type="string">propagationOrganizationCommunityMap</PARAM>
|
|
<PARAM managedBy="system" name="parameterValue" required="true" type="string">
|
|
{
|
|
"20|openorgs____::817280e710651751433d869c71f6eee2": ["mes"],
|
|
"20|openorgs____::809457868cca956bddde9c74e3a32c3c": ["mes"],
|
|
"20|openorgs____::cb386de57b0aa53d5bbf016e15720053": ["mes"],
|
|
"20|openorgs____::fb2e1f77e2506054fa25fef7eac7b685": ["mes"],
|
|
"20|openorgs____::c4ff9a638850ebdc39e28897b58166bc": ["mes"],
|
|
"20|openorgs____::00010e003d3ce9c05e038cadfb6d4f03": ["mes"],
|
|
"20|openorgs____::c20c22d6210eea23b58869029c2abe24": ["mes"],
|
|
"20|openorgs____::ad78005de675860b9f96b84d50fc19f6": ["mes"],
|
|
"20|openorgs____::a6bfaa7b9934dd8459ca94deac34c127": ["mes"],
|
|
"20|openorgs____::e68796680c3203d0d708e576c1d55de6": ["mes"],
|
|
"20|openorgs____::41206374e08ed608c03f56fc21ec3023": ["mes"],
|
|
"20|openorgs____::7dc15ac2260f80e90c849908afe20a6e": ["mes"],
|
|
"20|openorgs____::eaea0ef989e0b9245a5d50a6c519dc52": ["mes"],
|
|
"20|openorgs____::15fb2c0b223ca34ec651f34838d2daf9": ["mes"],
|
|
"20|openorgs____::3cb19734da640f9193b20291268c9a0e": ["mes"],
|
|
"20|openorgs____::da5a68a98b9ff7231d6c3c93cde988df": ["mes"],
|
|
"20|openorgs____::b436de29bae7906f94797b8a0c235585": ["mes"],
|
|
"20|openorgs____::6a5fcbfe36fed5e8b2b50128454cc4f0": ["mes"],
|
|
"20|openorgs____::3053c65910d1c064799cb9a93e57e5cd": ["mes"],
|
|
"20|openorgs____::f95684edab7a22ac87ea2ed7c47897f2": ["mes"],
|
|
"20|openorgs____::581dcea989b861fa0106d4874ecf2d66": ["beopen"],
|
|
"20|openorgs____::90a0f7c99fb72cd0e014fcdd38c08719": ["beopen"],
|
|
"20|openorgs____::11f6b2617abf37fe7193557d77d8cd00": ["beopen"],
|
|
"20|openorgs____::600c7afdde615a68e45cfceaf684782d": ["beopen"],
|
|
"20|openorgs____::ca4e3e4e6767e05b0828ef5f0cdc7292": ["beopen"],
|
|
"20|openorgs____::b79247e30e30a8f8532a30e3b816cda9": ["beopen"],
|
|
"20|openorgs____::9cb5ffc315d7bf0f97b2f0fdc37612aa": ["beopen"],
|
|
"20|openorgs____::ad863df6deda1619a25e7fad4a534891": ["beopen"],
|
|
"20|openorgs____::72162cfc2e7edaf7515c778e04d1952b": ["beopen"],
|
|
"20|openorgs____::9e29fb5b85151a6810ce7256b08475e4": [
|
|
"mes",
|
|
"beopen"
|
|
],
|
|
"20|openorgs____::9dd5545aacd3d8019e00c3f837269746": ["beopen"],
|
|
"20|openorgs____::a86e8b969264b4c92cbf79c289a3f61a": ["beopen"],
|
|
"20|openorgs____::d6b4b35b44951f55747a7139446d21a8": ["beopen"],
|
|
"20|openorgs____::8ec069b683b9e9387492ea0c6b88a806": ["beopen"],
|
|
"20|openorgs____::d11f981828c485cd23d93f7f24f24db1": ["eut", "tunet"],
|
|
"20|openorgs____::e66fe5dd092752e1dd6fd29fc699933a": ["eut"],
|
|
"20|openorgs____::526468206bca24c1c90da6a312295cf4": ["eut"],
|
|
"20|openorgs____::08e311e656e65ccb32e07c66b15b6ff7": ["eut"],
|
|
"20|openorgs____::55a1f889758964b77682904218fdb298": ["eut"],
|
|
"20|openorgs____::530092b6970d60a5329beb9f39e8d7d4": ["eut"],
|
|
"20|openorgs____::aadafa39392b3e200102596a3a4aad9d": ["eut"],
|
|
"20|openorgs____::c3fe999c74fad308132b8a5971367dce": ["eut"],
|
|
"20|openorgs____::1624ff7c01bb641b91f4518539a0c28a": ["aurora", "netherlands"],
|
|
"20|openorgs____::cdda7cfe17c89eb50628ec2eb1f8acd2": ["aurora"],
|
|
"20|openorgs____::818b75030e0e40612d69e049843ede7e": ["aurora"],
|
|
"20|openorgs____::0b0102bae51f4f4ef5ba57fbe1523b92": ["aurora"],
|
|
"20|openorgs____::d1f5d132714edad4a952e0414fc3c94b": ["aurora"],
|
|
"20|openorgs____::eb0669daa9efeb898a3090d8aac7c953": ["aurora"],
|
|
"20|openorgs____::eb391317ed0dc684aa81ac16265de041": ["aurora"],
|
|
"20|openorgs____::f7cfcc98245e22c7d6e321cde930e746": ["aurora"],
|
|
"20|openorgs____::f33179d3306ba2599f7a898b056b604f": ["aurora"],
|
|
"20|openorgs____::0a45de396fc864dc70d42d13f4e786e1": ["aurora"],
|
|
"20|openorgs____::039192f47416999e406a6b57da19beeb": ["aurora"],
|
|
"20|openorgs____::2023f56908812b5ac7045778fded8a0f": ["eutopia"],
|
|
"20|openorgs____::77f6d72a67be69ed181d83bc4a53fa54": ["eutopia"],
|
|
"20|openorgs____::b7e8d9a1ee2a31a5e9aaaaebac2d7cc3": ["eutopia"],
|
|
"20|openorgs____::be0ace0416ddcc3cc7ad6151d2d5d681": ["eutopia"],
|
|
"20|openorgs____::f6d0bdf4b935c2907d157429ad783eb1": ["eutopia"],
|
|
"20|openorgs____::6363a88f20cca34629ae690d539f9b67": ["eutopia"],
|
|
"20|openorgs____::52203596562cc56fcefb3e737c6e2441": ["eutopia"],
|
|
"20|openorgs____::6ba93bd765e2d7e5d2fb0cf3c1ac17b2": ["eutopia"],
|
|
"20|openorgs____::a6340e6ecf60f6bba163659df985b0f2": ["eutopia"],
|
|
"20|openorgs____::21731b21b089c81e0e1a5ed599316e2f": ["eutopia"],
|
|
"20|openorgs____::4efa936474b7478a534766170e2b3a78": ["forthem"],
|
|
"20|openorgs____::ff9f05849d91ca29be33aeeb07413536": ["forthem"],
|
|
"20|openorgs____::bfe8b338037acd41adfc0f100460e3db": ["forthem"],
|
|
"20|openorgs____::091f96b07fb8ad89e0732cd2f0c9bfcb": ["forthem"],
|
|
"20|openorgs____::5e4f5acff7b8362d8a431f27b0b123c3": ["forthem"],
|
|
"20|openorgs____::01ce6b73a6e390a709130c1bcfabeab9": ["forthem"],
|
|
"20|openorgs____::db9192d70ca3a941f6629fab56278fed": ["forthem"],
|
|
"20|openorgs____::a7502dd2669be577007c95445ca412fa": ["forthem"],
|
|
"20|openorgs____::09ddce938e9b0a54b5a831a1154be481": ["forthem"],
|
|
"20|openorgs____::7ac95584b6f0ae0b33027b9ce08cc96a": ["openaire-infrastructure"],
|
|
"20|openorgs____::027a83f256564db02650b6e1c24623d5": ["embrc"],
|
|
"20|openorgs____::dd8ba98db9a7a650207228be6f57ee29": ["instruct"],
|
|
"20|openorgs____::f0c3e27c112272a3781226c5890b228c": ["dariah"],
|
|
"20|openorgs____::0998a1d4dbfc5025ad963eeb2306969c": ["egi"],
|
|
"20|openorgs____::5ca2b95a1771149560222420df04abc7": ["sobigdata"],
|
|
"20|openorgs____::54d41926fa20801d03886c9685df4d38": ["epos"],
|
|
"20|openorgs____::ae84e04d1bb09ceec2f041532ad7175c": ["clarin"],
|
|
"20|openorgs____::5e91b35f34a6d4ead6432ef2f77d24bf": ["eu-conexus"],
|
|
"20|openorgs____::5e3400be408250ebb7d36b8d33bb3660": ["eu-conexus"],
|
|
"20|openorgs____::15ee7f73eb676be60ebc7629e42c7bf9": ["eu-conexus"],
|
|
"20|openorgs____::8a3fbaedd50cd6baabbba47b9976e4db": ["eu-conexus"],
|
|
"20|openorgs____::f1d25b27c8a254d0e0931db00497be26": ["eu-conexus"],
|
|
"20|openorgs____::b77c01aa15de3675da34277d48de2ec1": ["eu-conexus"],
|
|
"20|openorgs____::54cd984fc7d3b153ec2181f985041f02": ["eu-conexus", "tunet"],
|
|
"20|openorgs____::ed4f840895b2614ca59bc45ffb2eb15f": ["eu-conexus"],
|
|
"20|openorgs____::08c6812ca06b419e177b8e3949c9ec4c": ["eu-conexus"],
|
|
"20|openorgs____::3d57a5aadd2e0925bca78515278f2405": ["netherlands"],
|
|
"20|openorgs____::2ce10b2528220091bd0363c1e39394c1": ["netherlands"],
|
|
"20|openorgs____::fe49cd5c84da7a89da723e24cadeb99d": ["netherlands"],
|
|
"20|openorgs____::df96ffc1951dc9bc14b0695e3297f6e1": ["netherlands"],
|
|
"20|openorgs____::10020843b2d7fac7c04e40b55a631b0c": ["netherlands"],
|
|
"20|openorgs____::47f4bfbcef93eb928361dffaabd03b54": ["netherlands"],
|
|
"20|openorgs____::5e6bf8962665cdd040341171e5c631d8": ["netherlands"],
|
|
"20|openorgs____::79a0e60afef2e753b0dc12425ecb3f8c": ["netherlands"],
|
|
"20|openorgs____::a8767e80afb51d63802fdf1ddabeacb0": ["netherlands"],
|
|
"20|openorgs____::db787c50cf46f2b08e69f830e292e42d": ["netherlands"],
|
|
"20|openorgs____::2f735203eb40d8389a881e874bee537a": ["netherlands"],
|
|
"20|openorgs____::25b87dfee5c1855dab099c5a2eef225a": ["netherlands"],
|
|
"20|openorgs____::cb77737b5a390319030e6abdb12aa6ad": ["netherlands"],
|
|
"20|openorgs____::b0a51df36791d8bb939721e0ad1f92b5": ["netherlands"],
|
|
"20|openorgs____::e42580548da4a1d39bb67b60b971056e": ["netherlands"],
|
|
"20|openorgs____::2eeca4f78f2a04a35057c1fa7918e23b": ["netherlands"],
|
|
"20|openorgs____::a7e0018f6064f0dab7a8c9a48a61a1c6": ["netherlands"],
|
|
"20|openorgs____::ee938614ce34e199c2901282deb79db4": ["netherlands"],
|
|
"20|openorgs____::dd29a935a8c8b552752fd00524f5c2a3": ["netherlands"],
|
|
"20|openorgs____::fa2e191ad1d35db623f96abcedb9fb96": ["netherlands"],
|
|
"20|openorgs____::863c4184481f057bfe695c9c2786a90a": ["netherlands"],
|
|
"20|openorgs____::f2a84eb5c508838ba5dd728a1b8ac5ba": ["netherlands"],
|
|
"20|openorgs____::eb2f1247784acef155801d95b5ac937b": ["netherlands"],
|
|
"20|openorgs____::19274b9a576d284a2b900e2f112f6a0c": ["netherlands"],
|
|
"20|openorgs____::48ffea8c900e79deaa2256aa2c0a0ee2": ["netherlands"],
|
|
"20|openorgs____::54f2e88f3eb801dc7e49a4ca90fdd1b6": ["netherlands"],
|
|
"20|openorgs____::cb04606c0c4eb1696cf4bc74dba44bdf": ["netherlands"],
|
|
"20|openorgs____::a3af79fec4d09764e56cd6d4df1d976a": ["netherlands"],
|
|
"20|openorgs____::6c50ead5f8f9cac29c4c3066df9fba45": ["netherlands"],
|
|
"20|openorgs____::40a6eddd9af7a097cb83adb7c7b7fe55": ["netherlands"],
|
|
"20|openorgs____::0a49c26b60b634e6f8f6334bb8e078ae": ["netherlands"],
|
|
"20|openorgs____::374bcee2ff43200e9afe71b6dc606e6c": ["netherlands"],
|
|
"20|openorgs____::14e84ae5ab9b8fd2b05b65c19551a416": ["netherlands"],
|
|
"20|openorgs____::7d8ba78b303d8a0065029dffb414d4ae": ["netherlands"],
|
|
"20|openorgs____::b12b585c23a2b443b35cb33f57309a34": ["netherlands"],
|
|
"20|openorgs____::be5a09d4c12d29681a6985dd892f9f44": ["netherlands"],
|
|
"20|openorgs____::ba45682d5fd11bfdb695ca2fe5a0d377": ["netherlands"],
|
|
"20|openorgs____::58f65ed1ce3c9166e9c5f939bfdbf83a": ["netherlands"],
|
|
"20|openorgs____::81371ea94b1a09d3243e73d6ec3527ec": ["netherlands"],
|
|
"20|openorgs____::604881198363fedbb5d5478f465305f2": ["netherlands"],
|
|
"20|openorgs____::81b64115eb383a27a9c1820a2c760c89": ["netherlands"],
|
|
"20|openorgs____::ad8f981707a4a1d6c9e9ee60c02b9a11": ["netherlands"],
|
|
"20|openorgs____::994d7f6fc25c5de47b9212bb69524380": ["netherlands"],
|
|
"20|openorgs____::57a751ebab17a6996ba6836a89548f35": ["netherlands"],
|
|
"20|openorgs____::e79c2bf184a99250523a378626875955": ["netherlands"],
|
|
"20|openorgs____::05ba90eee0e9eef094a77240339d0ee5": ["netherlands"],
|
|
"20|openorgs____::04a66697fceec5d5e3bbf19b85135356": ["netherlands"],
|
|
"20|openorgs____::bb88336ade9d2deeece4dee6262054b4": ["netherlands"],
|
|
"20|openorgs____::b232880c47fb92a22ffea8e2e749e0ac": ["netherlands"],
|
|
"20|openorgs____::4aebed7bd67813bb8a325eb410a392be": ["netherlands"],
|
|
"20|openorgs____::4b2a9d306955ede352ba267af8616375": ["netherlands"],
|
|
"20|openorgs____::2d77f3a9829f7a9651ec60f597b479cd": ["netherlands"],
|
|
"20|openorgs____::2885352ab1a441a9d387424888e30796": ["netherlands"],
|
|
"20|openorgs____::95e5f0e3696306902a6749c406bb2728": ["netherlands"],
|
|
"20|openorgs____::048a44d990fd8a4710eef3a9e375360b": ["netherlands"],
|
|
"20|openorgs____::c254f2d5f6fdb12ecb37cfee25a3c4b1": ["netherlands"],
|
|
"20|openorgs____::775eab3c1281cb91d53a31c4a1ba1090": ["netherlands"],
|
|
"20|openorgs____::5f081781d2510771c1c6686c4a37d1a1": ["netherlands"],
|
|
"20|openorgs____::3f27054a1ab469826b9d3a1915452f89": ["netherlands"],
|
|
"20|openorgs____::0ac7645722cfee8253ce68cd2ab609c6": ["netherlands"],
|
|
"20|openorgs____::913ef94a4bc903f936164b00f287defa": ["netherlands"],
|
|
"20|openorgs____::7ec8de16e11c7fc9f7e4e32923a42b22": ["netherlands"],
|
|
"20|openorgs____::8a1b859747bd305bc855914588006010": ["netherlands"],
|
|
"20|openorgs____::d0a66a264d47d18baba1821d678627f3": ["netherlands"],
|
|
"20|openorgs____::c143bad570ce1ce6c1c5956f5f2d96ee": ["netherlands"],
|
|
"20|openorgs____::a5ac6dbe326433376c900329853d914a": ["netherlands"],
|
|
"20|openorgs____::de44646096a724bdba876f7f573e74eb": ["netherlands"],
|
|
"20|openorgs____::74691942ddde3989519f008b551ebb35": ["netherlands"],
|
|
"20|openorgs____::8cf49add8601037458e4ecdbd63a1c42": ["netherlands"],
|
|
"20|openorgs____::e086dd85625f9fe8b89dfc2d81441664": ["netherlands"],
|
|
"20|openorgs____::06b2bdd4093a09b5f6c33521a7bf665c": ["netherlands"],
|
|
"20|openorgs____::a08a62c3e21dd693af009638e21aba11": ["netherlands"],
|
|
"20|openorgs____::0e7e1aa241c64ac17aee64c0be4be069": ["netherlands"],
|
|
"20|openorgs____::20e491a3310e7146d52161c825a980ec": ["netherlands"],
|
|
"20|openorgs____::43e6dc2e3207f47c82560767ed928bab": ["netherlands"],
|
|
"20|openorgs____::919801f3040011bbef2dd28020917c7f": ["netherlands"],
|
|
"20|openorgs____::58d238642a71a8cf41bd68c3e4189151": ["netherlands"],
|
|
"20|openorgs____::7ef128431ed7a002d362e73a048d3fd5": ["netherlands"],
|
|
"20|openorgs____::743576a59064697f297fba7e07d6e0fe": ["netherlands"],
|
|
"20|openorgs____::3c2c37b7745ae3202347c3fa594bc66a": ["netherlands"],
|
|
"20|openorgs____::51609e11886740e4cf5f77d0f516a43c": ["netherlands"],
|
|
"20|openorgs____::e1c228979333191a8f4bd6b5f5d01644": ["netherlands"],
|
|
"20|openorgs____::1eaec9171998c0db1b627afe4a969430": ["netherlands"],
|
|
"20|openorgs____::4bfe0847ab9fa240154a274f13fa7ce0": ["netherlands"],
|
|
"20|openorgs____::917ca88ffc8c464dc902998c0d89c24e": ["netherlands"],
|
|
"20|openorgs____::990dcc0a2b128304fdde55df641a47b4": ["netherlands"],
|
|
"20|openorgs____::bc3c03a56c8ac610e083f74395896ed4": ["netherlands"],
|
|
"20|openorgs____::e2d0ca0969cf3610d277907a57b8ae7e": ["netherlands"],
|
|
"20|openorgs____::1820f7ff911f4cf656fd63fd17b2fea8": ["netherlands"],
|
|
"20|openorgs____::30f7a1b7ffaca5405d4fc22b45b346f8": ["netherlands"],
|
|
"20|openorgs____::64ff9bf918bf9552b30ba878bbf3eba6": ["netherlands"],
|
|
"20|openorgs____::6a6ff788c9f916d156e88942155543c8": ["netherlands"],
|
|
"20|openorgs____::938a3404eacaedfe1b192d39178606eb": ["netherlands"],
|
|
"20|openorgs____::a8369fe646a573e841439d2c24396b49": ["netherlands"],
|
|
"20|openorgs____::af7d4ec9149a9d4f43affddd3a5f6ade": ["netherlands"],
|
|
"20|openorgs____::b411400e9cb43b74991acf165fdbaf42": ["netherlands"],
|
|
"20|openorgs____::e9afeb0f06cd2672f64eba5c50b58508": ["netherlands"],
|
|
"20|openorgs____::f58b02a65440082244bedce6c5edddee": ["netherlands"],
|
|
"20|openorgs____::1a2cec15d21fef305e2f185f45ec866e": ["netherlands"],
|
|
"20|openorgs____::2310574386b83b200559887761a0e6b0": ["netherlands"],
|
|
"20|openorgs____::39501045bd561c8e721966822a5e76bb": ["netherlands"],
|
|
"20|openorgs____::468e82b8147d8b607a4820d954e8092c": ["netherlands"],
|
|
"20|openorgs____::51aec3d04ac8acf29e7d0cc01e85d49d": ["netherlands"],
|
|
"20|openorgs____::52f890d61aa7be9be055cb220264bb56": ["netherlands"],
|
|
"20|openorgs____::52fa090596519ef4cc00ee6d01dafa5e": ["netherlands"],
|
|
"20|openorgs____::546b0193ea282a484f5ffba1cf72c31a": ["netherlands"],
|
|
"20|openorgs____::55137182dc586d7fbcbd0651083cfcaa": ["netherlands"],
|
|
"20|openorgs____::5543c511cc8eeee82f81d5841adaf0c3": ["netherlands"],
|
|
"20|openorgs____::60c2723e09a04c216d34506524c4833c": ["netherlands"],
|
|
"20|openorgs____::63df05ca62ff76a23d92aeee3cd08e83": ["netherlands"],
|
|
"20|openorgs____::6c7d3222989e73edcd396801036ac142": ["netherlands"],
|
|
"20|openorgs____::6e1e359b40468348dc462a5e6d0d2f4c": ["netherlands"],
|
|
"20|openorgs____::7f37484efc0f92cb9974774e58d355d9": ["netherlands"],
|
|
"20|openorgs____::88038a30254a7cbe80a16c8824b355f3": ["netherlands"],
|
|
"20|openorgs____::9a1b949c6af565ac3cc0846a5f7473b9": ["netherlands"],
|
|
"20|openorgs____::9c1db524c2294a2ed7df7420291cd434": ["netherlands"],
|
|
"20|openorgs____::a0f6de4744e6e49fd80e96e33846f7c9": ["netherlands"],
|
|
"20|openorgs____::a246b8c7dc06716086d9012c00dfed93": ["netherlands"],
|
|
"20|openorgs____::cd7fa52a4719d56f6393f01a7d0a74ce": ["netherlands"],
|
|
"20|openorgs____::ce11a4e92d5cf66ba65cc0e1b5994daf": ["netherlands"],
|
|
"20|openorgs____::ef725cb463f17996424c6f27d7183e1d": ["netherlands"],
|
|
"20|openorgs____::ff39371bb6f5f4484908ed48ea41ba00": ["netherlands"],
|
|
"20|openorgs____::1ed7562253b98ec918a4007a532a750c": ["netherlands"],
|
|
"20|openorgs____::221325bfd7557cd363138ee80a62a729": ["netherlands"],
|
|
"20|openorgs____::2927d186274099c1a3b26b2517e3c717": ["netherlands"],
|
|
"20|openorgs____::2b9422d5f5bac4aae99b24e61586d3d3": ["netherlands"],
|
|
"20|openorgs____::3ad482503246d0c6b356b3212947d0b9": ["netherlands"],
|
|
"20|openorgs____::42b878fb8ecaf9a40a2446179d4ac4a5": ["netherlands"],
|
|
"20|openorgs____::4de50ef535f9dfa948617ff3b12e192b": ["netherlands"],
|
|
"20|openorgs____::56ab3c45d37a3e0b6fd8657a202d6f9f": ["netherlands"],
|
|
"20|openorgs____::6ad8bf5ed17bbf8809c34472cf0e4b82": ["netherlands"],
|
|
"20|openorgs____::6d172d9141a159458a1794dcca4733f8": ["netherlands"],
|
|
"20|openorgs____::7150901687aad5b0346843fb4426f7e3": ["netherlands"],
|
|
"20|openorgs____::7bcd05e7e96471553d6f395ce8670455": ["netherlands"],
|
|
"20|openorgs____::a0811c572ed4100f50c16060b95e9ec8": ["netherlands"],
|
|
"20|openorgs____::bcf63d0cf192b2941f29bff28c7a9d6b": ["netherlands"],
|
|
"20|openorgs____::d3dd82a7d383eff0f4e0108e4ba2a0db": ["netherlands"],
|
|
"20|openorgs____::d6951a5a11a8a33c96bfb4a8ce80a8d6": ["netherlands"],
|
|
"20|openorgs____::dc993215d316a13417c8e56e23a25b87": ["netherlands"],
|
|
"20|openorgs____::df72cb8cb3ce7b3a2fc8cdb05188ee15": ["netherlands"],
|
|
"20|openorgs____::09827cce030ebf5e04e1c955532e57c1": ["netherlands"],
|
|
"20|openorgs____::1948def781ac9956fed96f81710351db": ["netherlands"],
|
|
"20|openorgs____::271c04c73c94df9a3fa8d45c586a2637": ["netherlands"],
|
|
"20|openorgs____::365d0c26f67165ea52d918cebc858817": ["netherlands"],
|
|
"20|openorgs____::5149d47489dad962ee077bf2587b62a2": ["netherlands"],
|
|
"20|openorgs____::6fcc9d96bbfa51a0d67921ad3cced500": ["netherlands"],
|
|
"20|openorgs____::806a351b3659612b064034e7255de413": ["netherlands"],
|
|
"20|openorgs____::a8d0b3f004237455d2650006b004f10a": ["netherlands"],
|
|
"20|openorgs____::ffddafb3850edee334c70abad176c506":["netherlands"],
|
|
"20|openorgs____::5d55fb216b14691cf68218daf5d78cd9":["tunet"],
|
|
"20|openorgs____::5352e8197bcc25b2b66bcc0788ae8b92":["tunet"],
|
|
"20|openorgs____::81317c69d1aad1574f1a39f145ffd335":["tunet"]
|
|
}</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="resumeFrom"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE isStart="true" name="setFundersBlacklist" type="SetEnvParameter">
|
|
<DESCRIPTION>Set a regex of funder shortnames to exclude from the project reference processing</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="parameterName" required="true" type="string">referenceextraction_project_fundingclass_blacklist_regex</PARAM>
|
|
<PARAM managedBy="user" name="parameterValue" required="true" type="string">^DFG::.*$|^CONICYT::.*$|^RSF::.*$|^SGOV::.*$|^GSRT::.*$|^MIUR::.*$|^INNOVIRIS::.*$|^RIF::.*$|^SFRS::.*$</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="resumeFrom"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE isJoin="true" name="resumeFrom" type="Selection">
|
|
<DESCRIPTION>Resume the workflow from a specific node</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM function="validValues(['aggregatorGraph', 'promoteActionsRaw', 'graphEntityGrouping', 'graphCleaningFirst', 'graphResolution', 'IIS', 'promoteActionsIIS', 'dedup', 'promoteActionsDedup', 'orcidPropagation', 'bulkTagging', 'affiliationPropagation', 'affiliationSemRelPropagation', 'communityOrganizationPropagation', 'resultProjectPropagation', 'communitySemrelPropagation', 'countryPropagation', 'graphCleaning', 'blacklistRelations'])" managedBy="user" name="selection" required="true" type="string">communityOrganizationPropagation</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC name="aggregatorGraph" to="aggregatorGraph"/>
|
|
<ARC name="promoteActionsRaw" to="promoteActionsRaw"/>
|
|
<ARC name="graphEntityGrouping" to="graphEntityGrouping"/>
|
|
<ARC name="graphCleaningFirst" to="graphCleaningFirst"/>
|
|
<ARC name="graphResolution" to="graphResolution"/>
|
|
<ARC name="IIS" to="select_IIS"/>
|
|
<ARC name="promoteActionsIIS" to="promoteActionsIIS"/>
|
|
<ARC name="dedup" to="duplicateScan"/>
|
|
<ARC name="promoteActionsDedup" to="promoteActionsDedup"/>
|
|
<ARC name="orcidPropagation" to="orcidPropagation"/>
|
|
<ARC name="bulkTagging" to="bulkTagging"/>
|
|
<ARC name="affiliationPropagation" to="affiliationPropagation"/>
|
|
<ARC name="affiliationSemRelPropagation" to="affiliationSemRelPropagation"/>
|
|
<ARC name="communityOrganizationPropagation" to="communityOrganizationPropagation"/>
|
|
<ARC name="resultProjectPropagation" to="resultProjectPropagation"/>
|
|
<ARC name="communitySemrelPropagation" to="communitySemrelPropagation"/>
|
|
<ARC name="countryPropagation" to="countryPropagation"/>
|
|
<ARC name="graphCleaning" to="graphCleaning"/>
|
|
<ARC name="blacklistRelations" to="blacklistRelations"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="aggregatorGraph" type="SubmitHadoopJob">
|
|
<DESCRIPTION>create the PROD AGGREGATOR graph</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'graphOutputPath' : 'aggregatorGraphPath',
|
|
'isLookupUrl' : 'isLookUpUrl',
|
|
'reuseDBClaims' : 'reuseDBClaims',
|
|
'reuseODFClaims' : 'reuseODFClaims',
|
|
'reuseOAFClaims' : 'reuseOAFClaims',
|
|
'reuseDB' : 'reuseDB',
|
|
'reuseDBOpenorgs' : 'reuseDBOpenorgs',
|
|
'reuseODF' : 'reuseODF',
|
|
'reuseODF_hdfs' : 'reuseODFhdfs',
|
|
'reuseOAF' : 'reuseOAF',
|
|
'reuseOAF_hdfs' : 'reuseOAFhdfs',
|
|
'contentPath' : 'contentPath',
|
|
'nsPrefixBlacklist' : 'nsPrefixBlacklist',
|
|
'shouldPatchRelations' : 'shouldPatchRelations',
|
|
'idMappingPath' : 'idMappingPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/raw_all/oozie_app',
|
|
'mongoURL' : 'mongodb://services.openaire.eu',
|
|
'mongoDb' : 'mdstore',
|
|
'mdstoreManagerUrl' : 'https://services.openaire.eu/mdstoremanager',
|
|
'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus',
|
|
'postgresUser' : 'dnet',
|
|
'postgresPassword' : 'dnetPwd',
|
|
'postgresOpenOrgsURL' : 'jdbc:postgresql://10.19.65.39:5432/oa_organizations',
|
|
'postgresOpenOrgsUser' : 'oa_organizations',
|
|
'postgresOpenOrgsPassword' : 'tei1igiLaPheeghu',
|
|
'shouldHashId' : 'true',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/prod_aggregator'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="aggregatorGraph_continue"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="aggregatorGraph_continue" type="Selection">
|
|
<DESCRIPTION>Continue after aggregator graph?</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC name="NO" to="success"/>
|
|
<ARC name="YES" to="promoteActionsRaw"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="promoteActionsRaw" type="SubmitHadoopJob">
|
|
<DESCRIPTION>create the RAW graph</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'inputActionSetIds' : 'actionSetIdsRawGraph',
|
|
'inputGraphRootPath' : 'aggregatorGraphPath',
|
|
'outputGraphRootPath' : 'rawGraphPath',
|
|
'isLookupUrl' : 'isLookUpUrl'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/actionmanager/wf/main/oozie_app',
|
|
'sparkExecutorCores' : '3',
|
|
'sparkExecutorMemory' : '10G',
|
|
'activePromoteDatasetActionPayload' : 'true',
|
|
'activePromoteDatasourceActionPayload' : 'true',
|
|
'activePromoteOrganizationActionPayload' : 'true',
|
|
'activePromoteOtherResearchProductActionPayload' : 'true',
|
|
'activePromoteProjectActionPayload' : 'true',
|
|
'activePromotePublicationActionPayload' : 'true',
|
|
'activePromoteRelationActionPayload' : 'true',
|
|
'activePromoteResultActionPayload' : 'true',
|
|
'activePromoteSoftwareActionPayload' : 'true',
|
|
'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET',
|
|
'shouldGroupById':'false',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/promoteActionsRaw'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="promoteActionsRaw_continue"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="promoteActionsRaw_continue" type="Selection">
|
|
<DESCRIPTION>Continue after RAW graph?</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC name="NO" to="success"/>
|
|
<ARC name="YES" to="graphEntityGrouping"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="graphEntityGrouping" type="SubmitHadoopJob">
|
|
<DESCRIPTION>Graph grouping</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'graphBasePath':'rawGraphPath',
|
|
'graphOutputPath':'groupedGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/group/oozie_app',
|
|
'workingPath' : '/tmp/prod_provision/working_dir/grouping',
|
|
'sparkExecutorCores' : '4',
|
|
'sparkExecutorMemory' : '7G'
|
|
}
|
|
</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="graphEntityGrouping_continue"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="graphEntityGrouping_continue" type="Selection">
|
|
<DESCRIPTION>Continue after grouped graph?</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC name="NO" to="success"/>
|
|
<ARC name="YES" to="graphCleaningFirst"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="graphCleaningFirst" type="SubmitHadoopJob">
|
|
<DESCRIPTION>clean the properties in the graph typed as Qualifier according to the vocabulary indicated in schemeid</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'graphInputPath' : 'groupedGraphPath',
|
|
'graphOutputPath': 'cleanedFirstGraphPath',
|
|
'isLookupUrl': 'isLookUpUrl'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/clean/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/clean',
|
|
'shouldClean' : 'false'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="patchHostedBy"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="patchHostedBy" type="SubmitHadoopJob">
|
|
<DESCRIPTION>updates publication's hostedby info according to the ISSNs available from DOAJ and UNIBI</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'resumeFrom' : 'HBMresumeFrom',
|
|
'sourcePath' : 'cleanedFirstGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'hostedByMapPath' : '/user/dnet.production/data/hostedByMap',
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/hostedbymap/oozie_app',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '12G',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/hostedbymap',
|
|
'outputPath' : '/tmp/prod_provision/working_dir/hostedbymap',
|
|
'unibiFileURL':'https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv',
|
|
'doajJsonFileURL':'https://doaj.org/public-data-dump/journal',
|
|
'dumpPath':'/tmp/prod_provision/working_dir/hostedbymap',
|
|
'dumpFileName':'doaj_json.tar.gz'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="graphResolution"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="graphResolution" type="SubmitHadoopJob">
|
|
<DESCRIPTION>Resolve Relation</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'graphBasePath':'cleanedFirstGraphPath',
|
|
'unresolvedPath' :'unresolvedEntityPath',
|
|
'targetPath':'resolvedGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/resolution/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/relation_resolution',
|
|
'shouldCleanContext' : 'false',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '12G'
|
|
}
|
|
</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="select_IIS"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="select_IIS" type="Selection">
|
|
<DESCRIPTION>Shall we run IIS?</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC name="YES" to="prepareIISActionSets"/>
|
|
<ARC name="NO" to="promoteActionsIIS"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="prepareIISActionSets" type="PrepareActionSets">
|
|
<DESCRIPTION>prepare IIS action sets</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="sets" required="true" type="string">
|
|
[
|
|
{
|
|
'set' : 'iis-eosc-service-link',
|
|
'jobProperty' : 'export_action_set_id_document_eoscServices',
|
|
'enablingProperty' : 'active_referenceextraction_eoscservice',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-document-affiliation',
|
|
'jobProperty' : 'export_action_set_id_matched_doc_organizations',
|
|
'enablingProperty' : 'active_document_affiliation',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-referenced-projects-main',
|
|
'jobProperty' : 'export_action_set_id_document_referencedProjects',
|
|
'enablingProperty' : 'active_referenceextraction_project',
|
|
'enabled' : 'true'
|
|
},
|
|
{
|
|
'set' : 'iis-referenced-datasets-main',
|
|
'jobProperty' : 'export_action_set_id_document_referencedDatasets',
|
|
'enablingProperty' : 'active_referenceextraction_dataset',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-researchinitiative',
|
|
'jobProperty' : 'export_action_set_id_document_research_initiative',
|
|
'enablingProperty' : 'active_referenceextraction_researchinitiative',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-document-similarities',
|
|
'jobProperty' : 'export_action_set_id_document_similarities_standard',
|
|
'enablingProperty' : 'active_documentssimilarity',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-document-classes',
|
|
'jobProperty' : 'export_action_set_id_document_classes',
|
|
'enablingProperty' : 'active_documentsclassification',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-document-citations',
|
|
'jobProperty' : 'export_action_set_id_document_referencedDocuments',
|
|
'enablingProperty' : 'active_citationmatching',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-document-citations-relations',
|
|
'jobProperty' : 'export_action_set_id_citation_relations',
|
|
'enablingProperty' : 'active_citationmatching_relations',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-referenceextraction-pdb',
|
|
'jobProperty' : 'export_action_set_id_document_pdb',
|
|
'enablingProperty' : 'active_referenceextraction_pdb',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'document_software_url',
|
|
'jobProperty' : 'export_action_set_id_document_software_url',
|
|
'enablingProperty' : 'active_referenceextraction_software_url',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-entities-software',
|
|
'jobProperty' : 'export_action_set_id_entity_software',
|
|
'enablingProperty' : 'active_referenceextraction_software_url',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-communities',
|
|
'jobProperty' : 'export_action_set_id_document_community',
|
|
'enablingProperty' : 'active_referenceextraction_community',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-referenced-patents',
|
|
'jobProperty' : 'export_action_set_id_document_patent',
|
|
'enablingProperty' : 'active_referenceextraction_patent',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-entities-patent',
|
|
'jobProperty' : 'export_action_set_id_entity_patent',
|
|
'enablingProperty' : 'active_referenceextraction_patent',
|
|
'enabled' : 'false'
|
|
},
|
|
{
|
|
'set' : 'iis-covid-19',
|
|
'jobProperty' : 'export_action_set_id_document_covid19',
|
|
'enablingProperty' : 'active_referenceextraction_covid19',
|
|
'enabled' : 'false'
|
|
}
|
|
]
|
|
</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="prepareIISParameters"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="prepareIISParameters" type="PrepareIisMainParamsV2">
|
|
<DESCRIPTION>prepare IIS parameters</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="islookupLocationParam" required="true" type="string">import_islookup_service_location</PARAM>
|
|
<PARAM managedBy="system" name="objectStoreParam" required="true" type="string">import_content_objectstores_csv</PARAM>
|
|
<PARAM managedBy="system" name="objectStoreLocationParam" required="true" type="string">import_content_object_store_location</PARAM>
|
|
<PARAM managedBy="system" name="mdStoreStoreLocationParam" required="true" type="string">import_mdstore_service_location</PARAM>
|
|
<PARAM managedBy="system" name="mdStoreDatasetParam" required="true" type="string">import_dataset_mdstore_ids_csv</PARAM>
|
|
<PARAM managedBy="system" name="oozieWfAppPathParam" required="true" type="string">oozie.wf.application.path</PARAM>
|
|
<PARAM managedBy="user" name="oozieWfAppPath" required="true" type="string">/lib/iis/primary/snapshots/2023-05-24_dhp-schemas_2.13.1-patched</PARAM>
|
|
<PARAM function="validValues(['IIS','DM'])" managedBy="user" name="clusterName" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="user" name="importHbaseDumpLocation" required="true" type="string">deprecated - not used</PARAM>
|
|
<PARAM managedBy="system" name="importHbaseDumpLocationParam" required="true" type="string">import_infospace_graph_location</PARAM>
|
|
<PARAM managedBy="user" name="objectStoreBlacklistCSV" required="false" type="string"/>
|
|
<PARAM managedBy="system" name="importProjectConceptsContextCSVParam" required="true" type="string">import_project_concepts_context_ids_csv</PARAM>
|
|
<PARAM managedBy="user" name="importProjectConceptsContextCSV" required="true" type="string">argo-france,aurora,beopen,citizen-science,clarin,covid-19,dariah,dh-ch,dth,oa-pg,ebrains,egi,egrise,elixir-gr,embrc,enermaps,epos,eu-conexus,eut,eutopia,fet-fp7,fet-h2020,forthem,gotriple,heritage-science,inspired-ris,instruct,iperionhs,knowmad,lifewatch-eric,mes,neanias-atmospheric,neanias-underwater,neanias-space,netherlands,ni,north-american-studies,openaire-infrastructure,rda,risis,rural-digital-europe,science-innovation-policy,sdsn-gr,sobigdata,tunet</PARAM>
|
|
<PARAM managedBy="system" name="importServicesCollectedFromParam" required="true" type="string">import_infospace_eligible_service_collectedfrom_datasourceid</PARAM>
|
|
<PARAM managedBy="user" name="importServicesCollectedFrom" required="true" type="string">10|openaire____::2e06c1122c7df43765fdcf91080824fa</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="IIS_main"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="IIS_main" type="SubmitHadoopJob">
|
|
<DESCRIPTION>IIS main</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">iisMainJobV3</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'cluster' : 'cluster',
|
|
'oozie.wf.application.path' : 'oozie.wf.application.path',
|
|
'referenceextraction_project_fundingclass_blacklist_regex' : 'referenceextraction_project_fundingclass_blacklist_regex',
|
|
|
|
'active_document_affiliation' : 'active_document_affiliation',
|
|
'active_referenceextraction_project' : 'active_referenceextraction_project',
|
|
'active_referenceextraction_dataset' : 'active_referenceextraction_dataset',
|
|
'active_referenceextraction_researchinitiative' : 'active_referenceextraction_researchinitiative',
|
|
'active_documentsclassification' : 'active_documentsclassification',
|
|
'active_documentssimilarity' : 'active_documentssimilarity',
|
|
'active_citationmatching' : 'active_citationmatching',
|
|
'active_citationmatching_relations' : 'active_citationmatching_relations',
|
|
'active_referenceextraction_pdb' : 'active_referenceextraction_pdb',
|
|
'active_referenceextraction_software_url' : 'active_referenceextraction_software_url',
|
|
'active_referenceextraction_community' : 'active_referenceextraction_community',
|
|
'active_referenceextraction_patent' : 'active_referenceextraction_patent',
|
|
'active_referenceextraction_covid19' : 'active_referenceextraction_covid19',
|
|
'active_referenceextraction_eoscservice' : 'active_referenceextraction_eoscservice',
|
|
|
|
'import_content_objectstores_csv' : 'import_content_objectstores_csv',
|
|
'import_content_object_store_location' : 'import_content_object_store_location',
|
|
'import_mdstore_service_location' : 'import_mdstore_service_location',
|
|
'import_islookup_service_location' : 'import_islookup_service_location',
|
|
'import_project_concepts_context_ids_csv' : 'import_project_concepts_context_ids_csv',
|
|
'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv',
|
|
'import_infospace_graph_location' : 'resolvedGraphPath',
|
|
'import_infospace_eligible_service_collectedfrom_datasourceid' : 'import_infospace_eligible_service_collectedfrom_datasourceid',
|
|
|
|
'export_action_set_id_matched_doc_organizations' : 'export_action_set_id_matched_doc_organizations',
|
|
'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets',
|
|
'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects',
|
|
'export_action_set_id_document_research_initiative' : 'export_action_set_id_document_research_initiative',
|
|
'export_action_set_id_document_similarities_standard' : 'export_action_set_id_document_similarities_standard',
|
|
|
|
'export_action_set_id_document_referencedDocuments' : 'export_action_set_id_document_referencedDocuments',
|
|
'export_action_set_id_citation_relations' : 'export_action_set_id_citation_relations',
|
|
'export_action_set_id_document_pdb' : 'export_action_set_id_document_pdb',
|
|
'export_action_set_id_document_software_url' : 'export_action_set_id_document_software_url',
|
|
'export_action_set_id_entity_software' : 'export_action_set_id_entity_software',
|
|
'export_action_set_id_document_community' : 'export_action_set_id_document_community',
|
|
'export_action_set_id_document_patent' : 'export_action_set_id_document_patent',
|
|
'export_action_set_id_entity_patent' : 'export_action_set_id_entity_patent',
|
|
'export_action_set_id_document_covid19' : 'export_action_set_id_document_covid19',
|
|
'export_action_set_id_document_classes' : 'export_action_set_id_document_classes',
|
|
'export_action_set_id_document_eoscServices' : 'export_action_set_id_document_eoscServices'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'import_content_pdfaggregation_table_name' : 'pdfaggregation_i.payload'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="user" name="simulation" required="true" type="boolean">false</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="updateIISActionSets"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="updateIISActionSets" type="UpdateActionSets">
|
|
<DESCRIPTION>update IIS action sets</DESCRIPTION>
|
|
<PARAMETERS/>
|
|
<ARCS>
|
|
<ARC to="promoteActionsIIS"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="promoteActionsIIS" type="SubmitHadoopJob">
|
|
<DESCRIPTION>create the INFERRED graph</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'inputActionSetIds' : 'actionSetIdsIISGraph',
|
|
'inputGraphRootPath' : 'resolvedGraphPath',
|
|
'outputGraphRootPath' : 'inferredGraphPath',
|
|
'isLookupUrl' : 'isLookUpUrl'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/actionmanager/wf/main/oozie_app',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '12G',
|
|
'activePromoteDatasetActionPayload' : 'true',
|
|
'activePromoteDatasourceActionPayload' : 'true',
|
|
'activePromoteOrganizationActionPayload' : 'true',
|
|
'activePromoteOtherResearchProductActionPayload' : 'true',
|
|
'activePromoteProjectActionPayload' : 'true',
|
|
'activePromotePublicationActionPayload' : 'true',
|
|
'activePromoteRelationActionPayload' : 'true',
|
|
'activePromoteResultActionPayload' : 'true',
|
|
'activePromoteSoftwareActionPayload' : 'true',
|
|
'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET',
|
|
'shouldGroupById':'false',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/promoteActionsIIS'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="duplicateScan"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="duplicateScan" type="SubmitHadoopJob">
|
|
<DESCRIPTION>search for duplicates in the raw graph</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'actionSetId' : 'dedupConfig',
|
|
'graphBasePath' : 'inferredGraphPath',
|
|
'dedupGraphPath': 'dedupGraphPath',
|
|
'isLookUpUrl' : 'isLookUpUrl'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/dedup/scan/oozie_app',
|
|
'actionSetIdOpenorgs' : 'dedup-similarity-organization-simple',
|
|
'workingPath' : '/tmp/prod_provision/working_dir/dedup',
|
|
'whiteListPath' : '/data/dedup/whitelist_prod',
|
|
'sparkExecutorCores' : '3',
|
|
'sparkExecutorMemory' : '10G',
|
|
'sparkDriverMemory' : '6G'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="dedupConsistency"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="dedupConsistency" type="SubmitHadoopJob">
|
|
<DESCRIPTION>mark duplicates as deleted and redistribute the relationships</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'graphBasePath' : 'dedupGraphPath',
|
|
'graphOutputPath': 'consistentGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/dedup/consistency/oozie_app',
|
|
'workingPath' : '/tmp/prod_provision/working_dir/dedup',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '10G'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="promoteActionsDedup"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="promoteActionsDedup" type="SubmitHadoopJob">
|
|
<DESCRIPTION>applies the actionsets referring to deduped records</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'inputActionSetIds' : 'actionSetIdsDedup',
|
|
'inputGraphRootPath' : 'consistentGraphPath',
|
|
'outputGraphRootPath' : 'dedupEnrichedGraphPath',
|
|
'isLookupUrl' : 'isLookUpUrl'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/actionmanager/wf/main/oozie_app',
|
|
'sparkExecutorCores' : '3',
|
|
'sparkExecutorMemory' : '10G',
|
|
'activePromoteDatasetActionPayload' : 'true',
|
|
'activePromoteDatasourceActionPayload' : 'true',
|
|
'activePromoteOrganizationActionPayload' : 'true',
|
|
'activePromoteOtherResearchProductActionPayload' : 'true',
|
|
'activePromoteProjectActionPayload' : 'true',
|
|
'activePromotePublicationActionPayload' : 'true',
|
|
'activePromoteRelationActionPayload' : 'true',
|
|
'activePromoteResultActionPayload' : 'true',
|
|
'activePromoteSoftwareActionPayload' : 'true',
|
|
'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET',
|
|
'shouldGroupById':'true',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/promoteActionsDedup'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="promoteActionsDedup_continue"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="promoteActionsDedup_continue" type="Selection">
|
|
<DESCRIPTION>Continue after dedup enriched graph?</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC name="NO" to="success"/>
|
|
<ARC name="YES" to="orcidPropagation"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="orcidPropagation" type="SubmitHadoopJob">
|
|
<DESCRIPTION>propagates ORCID among results linked by allowedsemrels semantic relationships</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'dedupEnrichedGraphPath',
|
|
'outputPath': 'orcidGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/orcidtoresultfromsemrel/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/orcid',
|
|
'allowedsemrels' : 'IsSupplementedBy;IsSupplementTo;isSupplementedBy;isSupplementTo',
|
|
'saveGraph' : 'true',
|
|
'sparkExecutorCores' : '3',
|
|
'sparkExecutorMemory' : '10G'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="bulkTagging"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="bulkTagging" type="SubmitHadoopJob">
|
|
<DESCRIPTION>mark results respecting some rules as belonging to communities</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'orcidGraphPath',
|
|
'outputPath': 'bulkTaggingGraphPath',
|
|
'isLookUpUrl' : 'isLookUpUrl',
|
|
'pathMap' : 'bulkTaggingPathMap'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus',
|
|
'postgresUser' : 'dnet',
|
|
'postgresPassword' : 'dnetPwd',
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/bulktag/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/bulktag',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '10G'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="affiliationPropagation"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="affiliationPropagation" type="SubmitHadoopJob">
|
|
<DESCRIPTION>creates relashionships between results and organizations when the organizations are associated to institutional repositories</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'bulkTaggingGraphPath',
|
|
'outputPath': 'affiliationGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/affiliation/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/affiliation',
|
|
'saveGraph' : 'true',
|
|
'blacklist' : 'empty'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="affiliationSemRelPropagation"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="affiliationSemRelPropagation" type="SubmitHadoopJob">
|
|
<DESCRIPTION>creates relashionships between results and organizations when the organizations are in parent/child relationships. The childs to the parents</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'affiliationGraphPath',
|
|
'outputPath': 'affiliationSemRelGraphPath',
|
|
'iterations':'iterations'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/affiliation_semrel/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/affiliationsemrel',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '10G'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="communityOrganizationPropagation"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="communityOrganizationPropagation" type="SubmitHadoopJob">
|
|
<DESCRIPTION>marks as belonging to communities the result collected from datasources related to the organizations specified in the organizationCommunityMap</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'affiliationSemRelGraphPath',
|
|
'outputPath': 'communityOrganizationGraphPath',
|
|
'organizationtoresultcommunitymap': 'propagationOrganizationCommunityMap'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/community_organization/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/community_organization',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '12G',
|
|
'saveGraph' : 'true'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="resultProjectPropagation"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="resultProjectPropagation" type="SubmitHadoopJob">
|
|
<DESCRIPTION>created relation between projects and results linked to other results trough allowedsemrel semantic relations linked to projects</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'communityOrganizationGraphPath',
|
|
'outputPath': 'fundingGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/funding/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/funding',
|
|
'allowedsemrels' : 'IsSupplementedBy;IsSupplementTo',
|
|
'saveGraph' : 'true',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '10G'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="communitySemrelPropagation"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="communitySemrelPropagation" type="SubmitHadoopJob">
|
|
<DESCRIPTION>tag as belonging to communitites result in in allowedsemrels relation with other result already linked to communities </DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'fundingGraphPath',
|
|
'outputPath': 'communitySemRelGraphPath',
|
|
'isLookUpUrl' : 'isLookUpUrl'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/community_semrel/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/community_semrel',
|
|
'allowedsemrels' : 'IsSupplementedBy;IsSupplementTo',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '10G',
|
|
'saveGraph' : 'true'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="countryPropagation"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="countryPropagation" type="SubmitHadoopJob">
|
|
<DESCRIPTION>associated to results colleced from allowedtypes and those in the whithelist the country of the organization(s) handling the datasource it is collected from </DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'communitySemRelGraphPath',
|
|
'outputPath': 'countryGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/country/oozie_app',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '10G',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/country',
|
|
'allowedtypes' : 'pubsrepository::institutional',
|
|
'whitelist':'10|opendoar____::16e6a3326dd7d868cbc926602a61e4d0;10|openaire____::fdb035c8b3e0540a8d9a561a6c44f4de;10|eurocrisdris::fe4903425d9040f680d8610d9079ea14;10|openaire____::5b76240cc27a58c6f7ceef7d8c36660e;10|openaire____::172bbccecf8fca44ab6a6653e84cb92a;10|openaire____::149c6590f8a06b46314eed77bfca693f;10|eurocrisdris::a6026877c1a174d60f81fd71f62df1c1;10|openaire____::4692342f0992d91f9e705c26959f09e0;10|openaire____::8d529dbb05ec0284662b391789e8ae2a;10|openaire____::345c9d171ef3c5d706d08041d506428c;10|opendoar____::1c1d4df596d01da60385f0bb17a4a9e0;10|opendoar____::7a614fd06c325499f1680b9896beedeb;10|opendoar____::1ee3dfcd8a0645a25a35977997223d22;10|opendoar____::d296c101daa88a51f6ca8cfc1ac79b50;10|opendoar____::798ed7d4ee7138d49b8828958048130a;10|openaire____::c9d2209ecc4d45ba7b4ca7597acb88a2;10|eurocrisdris::c49e0fe4b9ba7b7fab717d1f0f0a674d;10|eurocrisdris::9ae43d14471c4b33661fedda6f06b539;10|eurocrisdris::432ca599953ff50cd4eeffe22faf3e48',
|
|
'saveGraph' : 'true'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="graphCleaning"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="graphCleaning" type="SubmitHadoopJob">
|
|
<DESCRIPTION>clean the properties in the graph typed as Qualifier according to the vocabulary indicated in schemeid</DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'graphInputPath' : 'countryGraphPath',
|
|
'graphOutputPath': 'cleanedGraphPath',
|
|
'isLookupUrl': 'isLookUpUrl'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/clean/oozie_app',
|
|
'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus',
|
|
'postgresUser' : 'dnet',
|
|
'postgresPassword' : 'dnetPwd',
|
|
'verifyCountryParam' : '10.17632;10.5061',
|
|
'collectedfrom' : 'NARCIS',
|
|
'country' : 'NL',
|
|
'sparkExecutorCores' : '3',
|
|
'sparkExecutorMemory' : '10G',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/clean',
|
|
'shouldClean' : 'true'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="blacklistRelations"/>
|
|
</ARCS>
|
|
</NODE>
|
|
<NODE name="blacklistRelations" type="SubmitHadoopJob">
|
|
<DESCRIPTION>removes blacklisted relations </DESCRIPTION>
|
|
<PARAMETERS>
|
|
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
|
|
<PARAM managedBy="system" name="cluster" required="true" type="string">IIS</PARAM>
|
|
<PARAM managedBy="system" name="envParams" required="true" type="string">
|
|
{
|
|
'sourcePath' : 'cleanedGraphPath',
|
|
'outputPath': 'blacklistedGraphPath'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="params" required="true" type="string">
|
|
{
|
|
'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/blacklist/oozie_app',
|
|
'workingDir' : '/tmp/prod_provision/working_dir/blacklist',
|
|
'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus',
|
|
'postgresUser' : 'dnet',
|
|
'postgresPassword' : 'dnetPwd',
|
|
'sparkExecutorCores' : '2',
|
|
'sparkExecutorMemory' : '10G'
|
|
}
|
|
</PARAM>
|
|
<PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">build-report</PARAM>
|
|
</PARAMETERS>
|
|
<ARCS>
|
|
<ARC to="success"/>
|
|
</ARCS>
|
|
</NODE>
|
|
</CONFIGURATION>
|
|
<STATUS>
|
|
<LAST_EXECUTION_ID>wf_20230605_132025_368</LAST_EXECUTION_ID>
|
|
<LAST_EXECUTION_DATE>2023-06-05T13:21:02+00:00</LAST_EXECUTION_DATE>
|
|
<LAST_EXECUTION_STATUS>FAILURE</LAST_EXECUTION_STATUS>
|
|
<LAST_EXECUTION_ERROR>eu.dnetlib.data.hadoop.rmi.HadoopServiceException: hadoop job: 0011965-221214081318119-oozie-oozi-W failed with status: KILLED, oozie log: 2023-06-05 13:20:37,012 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] No results found 2023-06-05 13:20:37,032 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] Start action [0011965-221214081318119-oozie-oozi-W@:start:] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:37,032 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] [***0011965-221214081318119-oozie-oozi-W@:start:***]Action status=DONE 2023-06-05 13:20:37,032 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] [***0011965-221214081318119-oozie-oozi-W@:start:***]Action updated in DB! 2023-06-05 13:20:37,078 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] No results found 2023-06-05 13:20:37,093 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@:start: 2023-06-05 13:20:37,093 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:37,140 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] Start action [0011965-221214081318119-oozie-oozi-W@reset_outputpath] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,251 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] [***0011965-221214081318119-oozie-oozi-W@reset_outputpath***]Action status=DONE 2023-06-05 13:20:48,251 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] [***0011965-221214081318119-oozie-oozi-W@reset_outputpath***]Action updated in DB! 2023-06-05 13:20:48,299 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] No results found 2023-06-05 13:20:48,322 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] Start action [0011965-221214081318119-oozie-oozi-W@copy_entities] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,322 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] [***0011965-221214081318119-oozie-oozi-W@copy_entities***]Action status=DONE 2023-06-05 13:20:48,322 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] [***0011965-221214081318119-oozie-oozi-W@copy_entities***]Action updated in DB! 2023-06-05 13:20:48,367 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,372 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,378 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,384 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,491 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] Start action [0011965-221214081318119-oozie-oozi-W@copy_organization] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,495 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] Start action [0011965-221214081318119-oozie-oozi-W@copy_relation] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,505 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] Start action [0011965-221214081318119-oozie-oozi-W@copy_datasources] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,513 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] Start action [0011965-221214081318119-oozie-oozi-W@copy_projects] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:49,676 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] checking action, hadoop job ID [job_1676554762303_45231] status [RUNNING] 2023-06-05 13:20:49,681 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] [***0011965-221214081318119-oozie-oozi-W@copy_relation***]Action status=RUNNING 2023-06-05 13:20:49,681 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] [***0011965-221214081318119-oozie-oozi-W@copy_relation***]Action updated in DB! 2023-06-05 13:20:49,692 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_relation 2023-06-05 13:20:49,717 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] checking action, hadoop job ID [job_1676554762303_45234] status [RUNNING] 2023-06-05 13:20:49,721 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] [***0011965-221214081318119-oozie-oozi-W@copy_projects***]Action status=RUNNING 2023-06-05 13:20:49,721 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] [***0011965-221214081318119-oozie-oozi-W@copy_projects***]Action updated in DB! 2023-06-05 13:20:49,733 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_projects 2023-06-05 13:20:49,764 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] checking action, hadoop job ID [job_1676554762303_45233] status [RUNNING] 2023-06-05 13:20:49,768 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] [***0011965-221214081318119-oozie-oozi-W@copy_organization***]Action status=RUNNING 2023-06-05 13:20:49,768 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] [***0011965-221214081318119-oozie-oozi-W@copy_organization***]Action updated in DB! 2023-06-05 13:20:49,771 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] checking action, hadoop job ID [job_1676554762303_45232] status [RUNNING] 2023-06-05 13:20:49,773 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_organization 2023-06-05 13:20:49,775 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] [***0011965-221214081318119-oozie-oozi-W@copy_datasources***]Action status=RUNNING 2023-06-05 13:20:49,775 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] [***0011965-221214081318119-oozie-oozi-W@copy_datasources***]Action updated in DB! 2023-06-05 13:20:49,779 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_datasources 2023-06-05 13:20:49,784 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_entities 2023-06-05 13:20:49,784 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@reset_outputpath 2023-06-05 13:20:53,821 INFO org.apache.oozie.command.wf.KillXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] STARTED WorkflowKillXCommand for jobId=0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:53,826 WARN org.apache.oozie.workflow.lite.LiteWorkflowInstance: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] workflow killed, killing [5] nodes 2023-06-05 13:20:53,866 INFO org.apache.oozie.command.wf.KillXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] ENDED WorkflowKillXCommand for jobId=0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:53,866 INFO org.apache.oozie.DagEngine: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] User claudio.atzori killed the WF job 0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:54,196 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_projects 2023-06-05 13:20:54,504 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_relation 2023-06-05 13:20:54,813 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_datasources 2023-06-05 13:20:55,126 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:55,126 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_organization </LAST_EXECUTION_ERROR>
|
|
<LAST_EXECUTION_OUTPUT name="mainlog:selection">communityOrganizationPropagation</LAST_EXECUTION_OUTPUT>
|
|
</STATUS>
|
|
</BODY>
|
|
</RESOURCE_PROFILE> |