From 340a577430625fd5e66052d14f498371e66e719a Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 28 Jun 2023 12:29:59 +0200 Subject: [PATCH] apis for propagationOrganizationCommunityMap --- .../community/CommunityApiController.java | 66 + .../community/db/CommunityService.java | 31 +- .../openaire/community/old_provision_wf.xml | 1251 +++++++++++++++++ 3 files changed, 1347 insertions(+), 1 deletion(-) create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/community/old_provision_wf.xml diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java index 4d04c401..7db0de6b 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java @@ -9,6 +9,8 @@ import static eu.dnetlib.openaire.common.ExporterConstants.R; import static eu.dnetlib.openaire.common.ExporterConstants.W; import java.util.List; +import java.util.Map; +import java.util.Set; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -523,4 +525,68 @@ public class CommunityApiController { res.setOpenAirecommunitylist(communityService.getOpenAIRECommunities(zenodoId)); return res; } + + // APIs to manage the propagationOrganizationCommunityMap + @RequestMapping(value = "/propagationOrganizationCommunityMap", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "Get the propagationOrganizationCommunityMap", description = "propagationOrganizationCommunityMap", tags = { + C_O, R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "not found"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public Map> getPropagationOrganizationCommunityMap() { + return communityService.getPropagationOrganizationCommunityMap(); + } + + @RequestMapping(value = "/community/{communityId}/propagationOrganizations", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "return the propagation organizations of a community", description = "return the propagation organizations of a community", tags = { + C_O, R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "not found"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public Set getPropagationOrganizationsForCommunity(@PathVariable final String communityId) { + return communityService.getPropagationOrganizationsForCommunity(communityId); + } + + @RequestMapping(value = "/community/{communityId}/propagationOrganizations", produces = { + "application/json" + }, method = RequestMethod.POST) + @Operation(summary = "add an organization to the propagationOrganizationCommunityMap", description = "add an organization to the propagationOrganizationCommunityMap", tags = { + C_O, W + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "not found"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public Set addPropagationOrganizationForCommunity(@PathVariable final String communityId, + @RequestParam final String organizationId) { + return communityService.addPropagationOrganizationForCommunity(communityId, organizationId); + } + + @RequestMapping(value = "/community/{communityId}/propagationOrganizations", produces = { + "application/json" + }, method = RequestMethod.DELETE) + @Operation(summary = "delete an organization to the propagationOrganizationCommunityMap", description = "delete an organization to the propagationOrganizationCommunityMap", tags = { + C_O, W + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "not found"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public Set removePropagationOrganizationForCommunity(@PathVariable final String communityId, + @RequestParam final String organizationId) { + return communityService.removePropagationOrganizationForCommunity(communityId, organizationId); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java index d9e10a05..fb310f0f 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java @@ -3,6 +3,7 @@ package eu.dnetlib.openaire.community.db; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Function; @@ -19,6 +20,7 @@ import org.springframework.stereotype.Service; import eu.dnetlib.openaire.community.db.model.DbCommunity; import eu.dnetlib.openaire.community.db.model.DbDatasource; import eu.dnetlib.openaire.community.db.model.DbDatasourcePK; +import eu.dnetlib.openaire.community.db.model.DbOrganization; import eu.dnetlib.openaire.community.db.model.DbProject; import eu.dnetlib.openaire.community.db.model.DbProjectPK; import eu.dnetlib.openaire.community.db.model.DbSubCommunity; @@ -49,7 +51,7 @@ public class CommunityService { // 1) Import tramite profili: // 1.1) Completare CommunityImporterController // 2) Subcommunities: visualizzazione tramite le context api - // 3) Come ritornare i campi delle community_orgs ? // Creare un metodo nuovo i valori devono arrivare dal wf di provision, esempio: + // 3) Importare le orgs dal wf di provision, ad esempio: // https://services.openaire.eu/is/mvc/ui/isManager.do#/profile/4801c33c-66ca-4ab6-af64-aa812194ec61_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl // 4) Verificare Tickets: #8835, #8854, #6483, #3259, #3494 @@ -321,4 +323,31 @@ public class CommunityService { return dbCommunityRepository.findByZenodoId(zenodoId); } + public Map> getPropagationOrganizationCommunityMap() { + return dbOrganizationRepository.findAll() + .stream() + .collect(Collectors.groupingBy(DbOrganization::getOrgId, Collectors.mapping(DbOrganization::getCommunity, Collectors.toSet()))); + } + + public Set getPropagationOrganizationsForCommunity(final String communityId) { + return dbOrganizationRepository.findByCommunity(communityId) + .stream() + .map(DbOrganization::getOrgId) + .collect(Collectors.toSet()); + } + + @Transactional + public Set addPropagationOrganizationForCommunity(final String communityId, final String organizationId) { + final DbOrganization o = new DbOrganization(communityId, organizationId); + dbOrganizationRepository.save(o); + return getPropagationOrganizationsForCommunity(communityId); + } + + @Transactional + public Set removePropagationOrganizationForCommunity(final String communityId, final String organizationId) { + final DbOrganization o = new DbOrganization(communityId, organizationId); + dbOrganizationRepository.delete(o); + return getPropagationOrganizationsForCommunity(communityId); + } + } diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/community/old_provision_wf.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/community/old_provision_wf.xml new file mode 100644 index 00000000..b4f4a27e --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/community/old_provision_wf.xml @@ -0,0 +1,1251 @@ + +
+ + + + + +
+ + Graph construction and processing + Data Provision + 30 + + + +- unresolvedEntityPath: path of unresolved entities +- nsPrefixBlacklist_PROD: blacklist of funder nsPrefixes from the prod aggregator +- idMappingPath: path of the map defining the relations id mappings +- dedupConfig: dedup orchestrator name +- actionSetIdsRawGraph: ActionSet ids to promote in the RAW graph +- isLookUpUrl: IS lookup service address +- iterations: number of iterations performed by the affiliation through semantic relation propagation algo + + + +{ + 'unresolvedEntityPath' : '/data/unresolved_PROD/content', + 'nsPrefixBlacklist' : 'conicytf____,dfgf________,gsrt________,innoviris___,miur________,rif_________,rsf_________,sgov________,sfrs________', + 'idMappingPath' : '/data/maps/funder_remapping/', + 'dedupConfig' : 'dedup-result-decisiontree-v3', + 'actionSetIdsRawGraph' : 'doiboost,orcidworks-no-doi,ror', + 'actionSetIdsIISGraph' : 'iis-researchinitiative,iis-document-citations,iis-document-citations-relations,iis-document-affiliation,iis-document-classes,iis-document-similarities,iis-referenced-datasets-main,iis-referenced-projects-main,iis-referenced-projects-backup,iis-referenceextraction-pdb,document_software_url,iis-extracted-metadata,iis-communities,iis-referenced-patents,iis-covid-19,opencitations,h2020classification,iis-entities-software,iis-entities-patent', + 'actionSetIdsDedup' : 'bipfinder-scores,usage_counts', + 'isLookUpUrl' : 'http://services.openaire.eu:8280/is/services/isLookUp?wsdl', + 'iterations' : '1', + + 'reuseDBClaims' : 'false', + 'reuseODFClaims' : 'false', + 'reuseOAFClaims' : 'false', + 'reuseODFhdfs' : 'false', + 'reuseOAFhdfs' : 'false', + 'reuseDB' : 'false', + 'reuseDBOpenorgs' : 'false', + 'reuseODF' : 'false', + 'reuseOAF' : 'false', + 'shouldPatchRelations' : 'true', + 'contentPath' : '/tmp/prod_aggregator', + + 'aggregatorGraphPath' : '/tmp/prod_provision/graph/00_graph_aggregator', + 'rawGraphPath' : '/tmp/prod_provision/graph/01_graph_raw', + 'groupedGraphPath' : '/tmp/prod_provision/graph/02_graph_grouped', + 'cleanedFirstGraphPath' : '/tmp/prod_provision/graph/03_graph_cleaned', + 'resolvedGraphPath' : '/tmp/prod_provision/graph/04_graph_resolved', + 'inferredGraphPath' : '/tmp/prod_provision/graph/05_graph_inferred', + 'dedupGraphPath' : '/tmp/prod_provision/graph/06_graph_dedup', + 'consistentGraphPath' : '/tmp/prod_provision/graph/07_graph_consistent', + 'dedupEnrichedGraphPath' : '/tmp/prod_provision/graph/08_graph_dedup_enriched', + 'orcidGraphPath' : '/tmp/prod_provision/graph/09_graph_orcid', + 'bulkTaggingGraphPath' : '/tmp/prod_provision/graph/10_graph_bulktagging', + 'affiliationGraphPath' : '/tmp/prod_provision/graph/11_graph_affiliation', + 'affiliationSemRelGraphPath' : '/tmp/prod_provision/graph/12_graph_affiliation_semantic_relation', + 'communityOrganizationGraphPath' : '/tmp/prod_provision/graph/13_graph_community_organization', + 'fundingGraphPath' : '/tmp/prod_provision/graph/14_graph_funding', + 'communitySemRelGraphPath' : '/tmp/prod_provision/graph/15_graph_community_semantic_relation', + 'countryGraphPath' : '/tmp/prod_provision/graph/16_graph_country', + 'cleanedGraphPath' : '/tmp/prod_provision/graph/17_graph_cleaned', + 'blacklistedGraphPath' : '/tmp/prod_provision/graph/18_graph_blacklisted' +} + + + + + + + + the step from which the hostedByMap step should be resumed. If the choice is DownloadBoth both the unibi gold and doaj files are downloaded anew. If the option id DownloadDoaj or DownloadGold only the selected file is downloaded. If the option id ProduceHBM a new hosted by map is produced. If the option is PrepareInfo the old HBM is used on the results and datasources. + + HBMresumeFrom + PrepareInfo + + + + + + + Set the map of paths for the Bulk Tagging + + bulkTaggingPathMap + {"author" : "$['author'][*]['fullname']", "title" : "$['title'][*]['value']","orcid":"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid')]['value']", "contributor" : "$['contributor'][*]['value']", "description" : "$['description'][*]['value']"} + + + + + + + Set the map of associations organization, community list for the propagation of + community to result through organization + + propagationOrganizationCommunityMap + +{ + "20|openorgs____::817280e710651751433d869c71f6eee2": ["mes"], + "20|openorgs____::809457868cca956bddde9c74e3a32c3c": ["mes"], + "20|openorgs____::cb386de57b0aa53d5bbf016e15720053": ["mes"], + "20|openorgs____::fb2e1f77e2506054fa25fef7eac7b685": ["mes"], + "20|openorgs____::c4ff9a638850ebdc39e28897b58166bc": ["mes"], + "20|openorgs____::00010e003d3ce9c05e038cadfb6d4f03": ["mes"], + "20|openorgs____::c20c22d6210eea23b58869029c2abe24": ["mes"], + "20|openorgs____::ad78005de675860b9f96b84d50fc19f6": ["mes"], + "20|openorgs____::a6bfaa7b9934dd8459ca94deac34c127": ["mes"], + "20|openorgs____::e68796680c3203d0d708e576c1d55de6": ["mes"], + "20|openorgs____::41206374e08ed608c03f56fc21ec3023": ["mes"], + "20|openorgs____::7dc15ac2260f80e90c849908afe20a6e": ["mes"], + "20|openorgs____::eaea0ef989e0b9245a5d50a6c519dc52": ["mes"], + "20|openorgs____::15fb2c0b223ca34ec651f34838d2daf9": ["mes"], + "20|openorgs____::3cb19734da640f9193b20291268c9a0e": ["mes"], + "20|openorgs____::da5a68a98b9ff7231d6c3c93cde988df": ["mes"], + "20|openorgs____::b436de29bae7906f94797b8a0c235585": ["mes"], + "20|openorgs____::6a5fcbfe36fed5e8b2b50128454cc4f0": ["mes"], + "20|openorgs____::3053c65910d1c064799cb9a93e57e5cd": ["mes"], + "20|openorgs____::f95684edab7a22ac87ea2ed7c47897f2": ["mes"], + "20|openorgs____::581dcea989b861fa0106d4874ecf2d66": ["beopen"], + "20|openorgs____::90a0f7c99fb72cd0e014fcdd38c08719": ["beopen"], + "20|openorgs____::11f6b2617abf37fe7193557d77d8cd00": ["beopen"], + "20|openorgs____::600c7afdde615a68e45cfceaf684782d": ["beopen"], + "20|openorgs____::ca4e3e4e6767e05b0828ef5f0cdc7292": ["beopen"], + "20|openorgs____::b79247e30e30a8f8532a30e3b816cda9": ["beopen"], + "20|openorgs____::9cb5ffc315d7bf0f97b2f0fdc37612aa": ["beopen"], + "20|openorgs____::ad863df6deda1619a25e7fad4a534891": ["beopen"], + "20|openorgs____::72162cfc2e7edaf7515c778e04d1952b": ["beopen"], + "20|openorgs____::9e29fb5b85151a6810ce7256b08475e4": [ + "mes", + "beopen" + ], + "20|openorgs____::9dd5545aacd3d8019e00c3f837269746": ["beopen"], + "20|openorgs____::a86e8b969264b4c92cbf79c289a3f61a": ["beopen"], + "20|openorgs____::d6b4b35b44951f55747a7139446d21a8": ["beopen"], + "20|openorgs____::8ec069b683b9e9387492ea0c6b88a806": ["beopen"], + "20|openorgs____::d11f981828c485cd23d93f7f24f24db1": ["eut", "tunet"], + "20|openorgs____::e66fe5dd092752e1dd6fd29fc699933a": ["eut"], + "20|openorgs____::526468206bca24c1c90da6a312295cf4": ["eut"], + "20|openorgs____::08e311e656e65ccb32e07c66b15b6ff7": ["eut"], + "20|openorgs____::55a1f889758964b77682904218fdb298": ["eut"], + "20|openorgs____::530092b6970d60a5329beb9f39e8d7d4": ["eut"], + "20|openorgs____::aadafa39392b3e200102596a3a4aad9d": ["eut"], + "20|openorgs____::c3fe999c74fad308132b8a5971367dce": ["eut"], + "20|openorgs____::1624ff7c01bb641b91f4518539a0c28a": ["aurora", "netherlands"], + "20|openorgs____::cdda7cfe17c89eb50628ec2eb1f8acd2": ["aurora"], + "20|openorgs____::818b75030e0e40612d69e049843ede7e": ["aurora"], + "20|openorgs____::0b0102bae51f4f4ef5ba57fbe1523b92": ["aurora"], + "20|openorgs____::d1f5d132714edad4a952e0414fc3c94b": ["aurora"], + "20|openorgs____::eb0669daa9efeb898a3090d8aac7c953": ["aurora"], + "20|openorgs____::eb391317ed0dc684aa81ac16265de041": ["aurora"], + "20|openorgs____::f7cfcc98245e22c7d6e321cde930e746": ["aurora"], + "20|openorgs____::f33179d3306ba2599f7a898b056b604f": ["aurora"], + "20|openorgs____::0a45de396fc864dc70d42d13f4e786e1": ["aurora"], + "20|openorgs____::039192f47416999e406a6b57da19beeb": ["aurora"], + "20|openorgs____::2023f56908812b5ac7045778fded8a0f": ["eutopia"], + "20|openorgs____::77f6d72a67be69ed181d83bc4a53fa54": ["eutopia"], + "20|openorgs____::b7e8d9a1ee2a31a5e9aaaaebac2d7cc3": ["eutopia"], + "20|openorgs____::be0ace0416ddcc3cc7ad6151d2d5d681": ["eutopia"], + "20|openorgs____::f6d0bdf4b935c2907d157429ad783eb1": ["eutopia"], + "20|openorgs____::6363a88f20cca34629ae690d539f9b67": ["eutopia"], + "20|openorgs____::52203596562cc56fcefb3e737c6e2441": ["eutopia"], + "20|openorgs____::6ba93bd765e2d7e5d2fb0cf3c1ac17b2": ["eutopia"], + "20|openorgs____::a6340e6ecf60f6bba163659df985b0f2": ["eutopia"], + "20|openorgs____::21731b21b089c81e0e1a5ed599316e2f": ["eutopia"], + "20|openorgs____::4efa936474b7478a534766170e2b3a78": ["forthem"], + "20|openorgs____::ff9f05849d91ca29be33aeeb07413536": ["forthem"], + "20|openorgs____::bfe8b338037acd41adfc0f100460e3db": ["forthem"], + "20|openorgs____::091f96b07fb8ad89e0732cd2f0c9bfcb": ["forthem"], + "20|openorgs____::5e4f5acff7b8362d8a431f27b0b123c3": ["forthem"], + "20|openorgs____::01ce6b73a6e390a709130c1bcfabeab9": ["forthem"], + "20|openorgs____::db9192d70ca3a941f6629fab56278fed": ["forthem"], + "20|openorgs____::a7502dd2669be577007c95445ca412fa": ["forthem"], + "20|openorgs____::09ddce938e9b0a54b5a831a1154be481": ["forthem"], + "20|openorgs____::7ac95584b6f0ae0b33027b9ce08cc96a": ["openaire-infrastructure"], + "20|openorgs____::027a83f256564db02650b6e1c24623d5": ["embrc"], + "20|openorgs____::dd8ba98db9a7a650207228be6f57ee29": ["instruct"], + "20|openorgs____::f0c3e27c112272a3781226c5890b228c": ["dariah"], + "20|openorgs____::0998a1d4dbfc5025ad963eeb2306969c": ["egi"], + "20|openorgs____::5ca2b95a1771149560222420df04abc7": ["sobigdata"], + "20|openorgs____::54d41926fa20801d03886c9685df4d38": ["epos"], + "20|openorgs____::ae84e04d1bb09ceec2f041532ad7175c": ["clarin"], + "20|openorgs____::5e91b35f34a6d4ead6432ef2f77d24bf": ["eu-conexus"], + "20|openorgs____::5e3400be408250ebb7d36b8d33bb3660": ["eu-conexus"], + "20|openorgs____::15ee7f73eb676be60ebc7629e42c7bf9": ["eu-conexus"], + "20|openorgs____::8a3fbaedd50cd6baabbba47b9976e4db": ["eu-conexus"], + "20|openorgs____::f1d25b27c8a254d0e0931db00497be26": ["eu-conexus"], + "20|openorgs____::b77c01aa15de3675da34277d48de2ec1": ["eu-conexus"], + "20|openorgs____::54cd984fc7d3b153ec2181f985041f02": ["eu-conexus", "tunet"], + "20|openorgs____::ed4f840895b2614ca59bc45ffb2eb15f": ["eu-conexus"], + "20|openorgs____::08c6812ca06b419e177b8e3949c9ec4c": ["eu-conexus"], + "20|openorgs____::3d57a5aadd2e0925bca78515278f2405": ["netherlands"], + "20|openorgs____::2ce10b2528220091bd0363c1e39394c1": ["netherlands"], + "20|openorgs____::fe49cd5c84da7a89da723e24cadeb99d": ["netherlands"], + "20|openorgs____::df96ffc1951dc9bc14b0695e3297f6e1": ["netherlands"], + "20|openorgs____::10020843b2d7fac7c04e40b55a631b0c": ["netherlands"], + "20|openorgs____::47f4bfbcef93eb928361dffaabd03b54": ["netherlands"], + "20|openorgs____::5e6bf8962665cdd040341171e5c631d8": ["netherlands"], + "20|openorgs____::79a0e60afef2e753b0dc12425ecb3f8c": ["netherlands"], + "20|openorgs____::a8767e80afb51d63802fdf1ddabeacb0": ["netherlands"], + "20|openorgs____::db787c50cf46f2b08e69f830e292e42d": ["netherlands"], + "20|openorgs____::2f735203eb40d8389a881e874bee537a": ["netherlands"], + "20|openorgs____::25b87dfee5c1855dab099c5a2eef225a": ["netherlands"], + "20|openorgs____::cb77737b5a390319030e6abdb12aa6ad": ["netherlands"], + "20|openorgs____::b0a51df36791d8bb939721e0ad1f92b5": ["netherlands"], + "20|openorgs____::e42580548da4a1d39bb67b60b971056e": ["netherlands"], + "20|openorgs____::2eeca4f78f2a04a35057c1fa7918e23b": ["netherlands"], + "20|openorgs____::a7e0018f6064f0dab7a8c9a48a61a1c6": ["netherlands"], + "20|openorgs____::ee938614ce34e199c2901282deb79db4": ["netherlands"], + "20|openorgs____::dd29a935a8c8b552752fd00524f5c2a3": ["netherlands"], + "20|openorgs____::fa2e191ad1d35db623f96abcedb9fb96": ["netherlands"], + "20|openorgs____::863c4184481f057bfe695c9c2786a90a": ["netherlands"], + "20|openorgs____::f2a84eb5c508838ba5dd728a1b8ac5ba": ["netherlands"], + "20|openorgs____::eb2f1247784acef155801d95b5ac937b": ["netherlands"], + "20|openorgs____::19274b9a576d284a2b900e2f112f6a0c": ["netherlands"], + "20|openorgs____::48ffea8c900e79deaa2256aa2c0a0ee2": ["netherlands"], + "20|openorgs____::54f2e88f3eb801dc7e49a4ca90fdd1b6": ["netherlands"], + "20|openorgs____::cb04606c0c4eb1696cf4bc74dba44bdf": ["netherlands"], + "20|openorgs____::a3af79fec4d09764e56cd6d4df1d976a": ["netherlands"], + "20|openorgs____::6c50ead5f8f9cac29c4c3066df9fba45": ["netherlands"], + "20|openorgs____::40a6eddd9af7a097cb83adb7c7b7fe55": ["netherlands"], + "20|openorgs____::0a49c26b60b634e6f8f6334bb8e078ae": ["netherlands"], + "20|openorgs____::374bcee2ff43200e9afe71b6dc606e6c": ["netherlands"], + "20|openorgs____::14e84ae5ab9b8fd2b05b65c19551a416": ["netherlands"], + "20|openorgs____::7d8ba78b303d8a0065029dffb414d4ae": ["netherlands"], + "20|openorgs____::b12b585c23a2b443b35cb33f57309a34": ["netherlands"], + "20|openorgs____::be5a09d4c12d29681a6985dd892f9f44": ["netherlands"], + "20|openorgs____::ba45682d5fd11bfdb695ca2fe5a0d377": ["netherlands"], + "20|openorgs____::58f65ed1ce3c9166e9c5f939bfdbf83a": ["netherlands"], + "20|openorgs____::81371ea94b1a09d3243e73d6ec3527ec": ["netherlands"], + "20|openorgs____::604881198363fedbb5d5478f465305f2": ["netherlands"], + "20|openorgs____::81b64115eb383a27a9c1820a2c760c89": ["netherlands"], + "20|openorgs____::ad8f981707a4a1d6c9e9ee60c02b9a11": ["netherlands"], + "20|openorgs____::994d7f6fc25c5de47b9212bb69524380": ["netherlands"], + "20|openorgs____::57a751ebab17a6996ba6836a89548f35": ["netherlands"], + "20|openorgs____::e79c2bf184a99250523a378626875955": ["netherlands"], + "20|openorgs____::05ba90eee0e9eef094a77240339d0ee5": ["netherlands"], + "20|openorgs____::04a66697fceec5d5e3bbf19b85135356": ["netherlands"], + "20|openorgs____::bb88336ade9d2deeece4dee6262054b4": ["netherlands"], + "20|openorgs____::b232880c47fb92a22ffea8e2e749e0ac": ["netherlands"], + "20|openorgs____::4aebed7bd67813bb8a325eb410a392be": ["netherlands"], + "20|openorgs____::4b2a9d306955ede352ba267af8616375": ["netherlands"], + "20|openorgs____::2d77f3a9829f7a9651ec60f597b479cd": ["netherlands"], + "20|openorgs____::2885352ab1a441a9d387424888e30796": ["netherlands"], + "20|openorgs____::95e5f0e3696306902a6749c406bb2728": ["netherlands"], + "20|openorgs____::048a44d990fd8a4710eef3a9e375360b": ["netherlands"], + "20|openorgs____::c254f2d5f6fdb12ecb37cfee25a3c4b1": ["netherlands"], + "20|openorgs____::775eab3c1281cb91d53a31c4a1ba1090": ["netherlands"], + "20|openorgs____::5f081781d2510771c1c6686c4a37d1a1": ["netherlands"], + "20|openorgs____::3f27054a1ab469826b9d3a1915452f89": ["netherlands"], + "20|openorgs____::0ac7645722cfee8253ce68cd2ab609c6": ["netherlands"], + "20|openorgs____::913ef94a4bc903f936164b00f287defa": ["netherlands"], + "20|openorgs____::7ec8de16e11c7fc9f7e4e32923a42b22": ["netherlands"], + "20|openorgs____::8a1b859747bd305bc855914588006010": ["netherlands"], + "20|openorgs____::d0a66a264d47d18baba1821d678627f3": ["netherlands"], + "20|openorgs____::c143bad570ce1ce6c1c5956f5f2d96ee": ["netherlands"], + "20|openorgs____::a5ac6dbe326433376c900329853d914a": ["netherlands"], + "20|openorgs____::de44646096a724bdba876f7f573e74eb": ["netherlands"], + "20|openorgs____::74691942ddde3989519f008b551ebb35": ["netherlands"], + "20|openorgs____::8cf49add8601037458e4ecdbd63a1c42": ["netherlands"], + "20|openorgs____::e086dd85625f9fe8b89dfc2d81441664": ["netherlands"], + "20|openorgs____::06b2bdd4093a09b5f6c33521a7bf665c": ["netherlands"], + "20|openorgs____::a08a62c3e21dd693af009638e21aba11": ["netherlands"], + "20|openorgs____::0e7e1aa241c64ac17aee64c0be4be069": ["netherlands"], + "20|openorgs____::20e491a3310e7146d52161c825a980ec": ["netherlands"], + "20|openorgs____::43e6dc2e3207f47c82560767ed928bab": ["netherlands"], + "20|openorgs____::919801f3040011bbef2dd28020917c7f": ["netherlands"], + "20|openorgs____::58d238642a71a8cf41bd68c3e4189151": ["netherlands"], + "20|openorgs____::7ef128431ed7a002d362e73a048d3fd5": ["netherlands"], + "20|openorgs____::743576a59064697f297fba7e07d6e0fe": ["netherlands"], + "20|openorgs____::3c2c37b7745ae3202347c3fa594bc66a": ["netherlands"], + "20|openorgs____::51609e11886740e4cf5f77d0f516a43c": ["netherlands"], + "20|openorgs____::e1c228979333191a8f4bd6b5f5d01644": ["netherlands"], + "20|openorgs____::1eaec9171998c0db1b627afe4a969430": ["netherlands"], + "20|openorgs____::4bfe0847ab9fa240154a274f13fa7ce0": ["netherlands"], + "20|openorgs____::917ca88ffc8c464dc902998c0d89c24e": ["netherlands"], + "20|openorgs____::990dcc0a2b128304fdde55df641a47b4": ["netherlands"], + "20|openorgs____::bc3c03a56c8ac610e083f74395896ed4": ["netherlands"], + "20|openorgs____::e2d0ca0969cf3610d277907a57b8ae7e": ["netherlands"], + "20|openorgs____::1820f7ff911f4cf656fd63fd17b2fea8": ["netherlands"], + "20|openorgs____::30f7a1b7ffaca5405d4fc22b45b346f8": ["netherlands"], + "20|openorgs____::64ff9bf918bf9552b30ba878bbf3eba6": ["netherlands"], + "20|openorgs____::6a6ff788c9f916d156e88942155543c8": ["netherlands"], + "20|openorgs____::938a3404eacaedfe1b192d39178606eb": ["netherlands"], + "20|openorgs____::a8369fe646a573e841439d2c24396b49": ["netherlands"], + "20|openorgs____::af7d4ec9149a9d4f43affddd3a5f6ade": ["netherlands"], + "20|openorgs____::b411400e9cb43b74991acf165fdbaf42": ["netherlands"], + "20|openorgs____::e9afeb0f06cd2672f64eba5c50b58508": ["netherlands"], + "20|openorgs____::f58b02a65440082244bedce6c5edddee": ["netherlands"], + "20|openorgs____::1a2cec15d21fef305e2f185f45ec866e": ["netherlands"], + "20|openorgs____::2310574386b83b200559887761a0e6b0": ["netherlands"], + "20|openorgs____::39501045bd561c8e721966822a5e76bb": ["netherlands"], + "20|openorgs____::468e82b8147d8b607a4820d954e8092c": ["netherlands"], + "20|openorgs____::51aec3d04ac8acf29e7d0cc01e85d49d": ["netherlands"], + "20|openorgs____::52f890d61aa7be9be055cb220264bb56": ["netherlands"], + "20|openorgs____::52fa090596519ef4cc00ee6d01dafa5e": ["netherlands"], + "20|openorgs____::546b0193ea282a484f5ffba1cf72c31a": ["netherlands"], + "20|openorgs____::55137182dc586d7fbcbd0651083cfcaa": ["netherlands"], + "20|openorgs____::5543c511cc8eeee82f81d5841adaf0c3": ["netherlands"], + "20|openorgs____::60c2723e09a04c216d34506524c4833c": ["netherlands"], + "20|openorgs____::63df05ca62ff76a23d92aeee3cd08e83": ["netherlands"], + "20|openorgs____::6c7d3222989e73edcd396801036ac142": ["netherlands"], + "20|openorgs____::6e1e359b40468348dc462a5e6d0d2f4c": ["netherlands"], + "20|openorgs____::7f37484efc0f92cb9974774e58d355d9": ["netherlands"], + "20|openorgs____::88038a30254a7cbe80a16c8824b355f3": ["netherlands"], + "20|openorgs____::9a1b949c6af565ac3cc0846a5f7473b9": ["netherlands"], + "20|openorgs____::9c1db524c2294a2ed7df7420291cd434": ["netherlands"], + "20|openorgs____::a0f6de4744e6e49fd80e96e33846f7c9": ["netherlands"], + "20|openorgs____::a246b8c7dc06716086d9012c00dfed93": ["netherlands"], + "20|openorgs____::cd7fa52a4719d56f6393f01a7d0a74ce": ["netherlands"], + "20|openorgs____::ce11a4e92d5cf66ba65cc0e1b5994daf": ["netherlands"], + "20|openorgs____::ef725cb463f17996424c6f27d7183e1d": ["netherlands"], + "20|openorgs____::ff39371bb6f5f4484908ed48ea41ba00": ["netherlands"], + "20|openorgs____::1ed7562253b98ec918a4007a532a750c": ["netherlands"], + "20|openorgs____::221325bfd7557cd363138ee80a62a729": ["netherlands"], + "20|openorgs____::2927d186274099c1a3b26b2517e3c717": ["netherlands"], + "20|openorgs____::2b9422d5f5bac4aae99b24e61586d3d3": ["netherlands"], + "20|openorgs____::3ad482503246d0c6b356b3212947d0b9": ["netherlands"], + "20|openorgs____::42b878fb8ecaf9a40a2446179d4ac4a5": ["netherlands"], + "20|openorgs____::4de50ef535f9dfa948617ff3b12e192b": ["netherlands"], + "20|openorgs____::56ab3c45d37a3e0b6fd8657a202d6f9f": ["netherlands"], + "20|openorgs____::6ad8bf5ed17bbf8809c34472cf0e4b82": ["netherlands"], + "20|openorgs____::6d172d9141a159458a1794dcca4733f8": ["netherlands"], + "20|openorgs____::7150901687aad5b0346843fb4426f7e3": ["netherlands"], + "20|openorgs____::7bcd05e7e96471553d6f395ce8670455": ["netherlands"], + "20|openorgs____::a0811c572ed4100f50c16060b95e9ec8": ["netherlands"], + "20|openorgs____::bcf63d0cf192b2941f29bff28c7a9d6b": ["netherlands"], + "20|openorgs____::d3dd82a7d383eff0f4e0108e4ba2a0db": ["netherlands"], + "20|openorgs____::d6951a5a11a8a33c96bfb4a8ce80a8d6": ["netherlands"], + "20|openorgs____::dc993215d316a13417c8e56e23a25b87": ["netherlands"], + "20|openorgs____::df72cb8cb3ce7b3a2fc8cdb05188ee15": ["netherlands"], + "20|openorgs____::09827cce030ebf5e04e1c955532e57c1": ["netherlands"], + "20|openorgs____::1948def781ac9956fed96f81710351db": ["netherlands"], + "20|openorgs____::271c04c73c94df9a3fa8d45c586a2637": ["netherlands"], + "20|openorgs____::365d0c26f67165ea52d918cebc858817": ["netherlands"], + "20|openorgs____::5149d47489dad962ee077bf2587b62a2": ["netherlands"], + "20|openorgs____::6fcc9d96bbfa51a0d67921ad3cced500": ["netherlands"], + "20|openorgs____::806a351b3659612b064034e7255de413": ["netherlands"], + "20|openorgs____::a8d0b3f004237455d2650006b004f10a": ["netherlands"], + "20|openorgs____::ffddafb3850edee334c70abad176c506":["netherlands"], + "20|openorgs____::5d55fb216b14691cf68218daf5d78cd9":["tunet"], + "20|openorgs____::5352e8197bcc25b2b66bcc0788ae8b92":["tunet"], + "20|openorgs____::81317c69d1aad1574f1a39f145ffd335":["tunet"] +} + + + + + + + Set a regex of funder shortnames to exclude from the project reference processing + + referenceextraction_project_fundingclass_blacklist_regex + ^DFG::.*$|^CONICYT::.*$|^RSF::.*$|^SGOV::.*$|^GSRT::.*$|^MIUR::.*$|^INNOVIRIS::.*$|^RIF::.*$|^SFRS::.*$ + + + + + + + Resume the workflow from a specific node + + communityOrganizationPropagation + + + + + + + + + + + + + + + + + + + + + + + + + create the PROD AGGREGATOR graph + + executeOozieJob + IIS + + { + 'graphOutputPath' : 'aggregatorGraphPath', + 'isLookupUrl' : 'isLookUpUrl', + 'reuseDBClaims' : 'reuseDBClaims', + 'reuseODFClaims' : 'reuseODFClaims', + 'reuseOAFClaims' : 'reuseOAFClaims', + 'reuseDB' : 'reuseDB', + 'reuseDBOpenorgs' : 'reuseDBOpenorgs', + 'reuseODF' : 'reuseODF', + 'reuseODF_hdfs' : 'reuseODFhdfs', + 'reuseOAF' : 'reuseOAF', + 'reuseOAF_hdfs' : 'reuseOAFhdfs', + 'contentPath' : 'contentPath', + 'nsPrefixBlacklist' : 'nsPrefixBlacklist', + 'shouldPatchRelations' : 'shouldPatchRelations', + 'idMappingPath' : 'idMappingPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/raw_all/oozie_app', + 'mongoURL' : 'mongodb://services.openaire.eu', + 'mongoDb' : 'mdstore', + 'mdstoreManagerUrl' : 'https://services.openaire.eu/mdstoremanager', + 'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus', + 'postgresUser' : 'dnet', + 'postgresPassword' : 'dnetPwd', + 'postgresOpenOrgsURL' : 'jdbc:postgresql://10.19.65.39:5432/oa_organizations', + 'postgresOpenOrgsUser' : 'oa_organizations', + 'postgresOpenOrgsPassword' : 'tei1igiLaPheeghu', + 'shouldHashId' : 'true', + 'workingDir' : '/tmp/prod_provision/working_dir/prod_aggregator' + } + + build-report + + + + + + + Continue after aggregator graph? + + YES + + + + + + + + create the RAW graph + + executeOozieJob + IIS + + { + 'inputActionSetIds' : 'actionSetIdsRawGraph', + 'inputGraphRootPath' : 'aggregatorGraphPath', + 'outputGraphRootPath' : 'rawGraphPath', + 'isLookupUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/actionmanager/wf/main/oozie_app', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', + 'activePromoteDatasetActionPayload' : 'true', + 'activePromoteDatasourceActionPayload' : 'true', + 'activePromoteOrganizationActionPayload' : 'true', + 'activePromoteOtherResearchProductActionPayload' : 'true', + 'activePromoteProjectActionPayload' : 'true', + 'activePromotePublicationActionPayload' : 'true', + 'activePromoteRelationActionPayload' : 'true', + 'activePromoteResultActionPayload' : 'true', + 'activePromoteSoftwareActionPayload' : 'true', + 'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET', + 'shouldGroupById':'false', + 'workingDir' : '/tmp/prod_provision/working_dir/promoteActionsRaw' + } + + build-report + + + + + + + Continue after RAW graph? + + YES + + + + + + + + Graph grouping + + executeOozieJob + IIS + + { + 'graphBasePath':'rawGraphPath', + 'graphOutputPath':'groupedGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/group/oozie_app', + 'workingPath' : '/tmp/prod_provision/working_dir/grouping', + 'sparkExecutorCores' : '4', + 'sparkExecutorMemory' : '7G' + } + + + + + + + + Continue after grouped graph? + + YES + + + + + + + + clean the properties in the graph typed as Qualifier according to the vocabulary indicated in schemeid + + executeOozieJob + IIS + + { + 'graphInputPath' : 'groupedGraphPath', + 'graphOutputPath': 'cleanedFirstGraphPath', + 'isLookupUrl': 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/clean/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/clean', + 'shouldClean' : 'false' + } + + build-report + + + + + + + updates publication's hostedby info according to the ISSNs available from DOAJ and UNIBI + + executeOozieJob + IIS + + { + 'resumeFrom' : 'HBMresumeFrom', + 'sourcePath' : 'cleanedFirstGraphPath' + } + + + { + 'hostedByMapPath' : '/user/dnet.production/data/hostedByMap', + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/hostedbymap/oozie_app', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '12G', + 'workingDir' : '/tmp/prod_provision/working_dir/hostedbymap', + 'outputPath' : '/tmp/prod_provision/working_dir/hostedbymap', + 'unibiFileURL':'https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv', + 'doajJsonFileURL':'https://doaj.org/public-data-dump/journal', + 'dumpPath':'/tmp/prod_provision/working_dir/hostedbymap', + 'dumpFileName':'doaj_json.tar.gz' + } + + build-report + + + + + + + Resolve Relation + + executeOozieJob + IIS + + { + 'graphBasePath':'cleanedFirstGraphPath', + 'unresolvedPath' :'unresolvedEntityPath', + 'targetPath':'resolvedGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/resolution/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/relation_resolution', + 'shouldCleanContext' : 'false', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '12G' + } + + + + + + + + Shall we run IIS? + + YES + + + + + + + + prepare IIS action sets + + + [ + { + 'set' : 'iis-eosc-service-link', + 'jobProperty' : 'export_action_set_id_document_eoscServices', + 'enablingProperty' : 'active_referenceextraction_eoscservice', + 'enabled' : 'false' + }, + { + 'set' : 'iis-document-affiliation', + 'jobProperty' : 'export_action_set_id_matched_doc_organizations', + 'enablingProperty' : 'active_document_affiliation', + 'enabled' : 'false' + }, + { + 'set' : 'iis-referenced-projects-main', + 'jobProperty' : 'export_action_set_id_document_referencedProjects', + 'enablingProperty' : 'active_referenceextraction_project', + 'enabled' : 'true' + }, + { + 'set' : 'iis-referenced-datasets-main', + 'jobProperty' : 'export_action_set_id_document_referencedDatasets', + 'enablingProperty' : 'active_referenceextraction_dataset', + 'enabled' : 'false' + }, + { + 'set' : 'iis-researchinitiative', + 'jobProperty' : 'export_action_set_id_document_research_initiative', + 'enablingProperty' : 'active_referenceextraction_researchinitiative', + 'enabled' : 'false' + }, + { + 'set' : 'iis-document-similarities', + 'jobProperty' : 'export_action_set_id_document_similarities_standard', + 'enablingProperty' : 'active_documentssimilarity', + 'enabled' : 'false' + }, + { + 'set' : 'iis-document-classes', + 'jobProperty' : 'export_action_set_id_document_classes', + 'enablingProperty' : 'active_documentsclassification', + 'enabled' : 'false' + }, + { + 'set' : 'iis-document-citations', + 'jobProperty' : 'export_action_set_id_document_referencedDocuments', + 'enablingProperty' : 'active_citationmatching', + 'enabled' : 'false' + }, + { + 'set' : 'iis-document-citations-relations', + 'jobProperty' : 'export_action_set_id_citation_relations', + 'enablingProperty' : 'active_citationmatching_relations', + 'enabled' : 'false' + }, + { + 'set' : 'iis-referenceextraction-pdb', + 'jobProperty' : 'export_action_set_id_document_pdb', + 'enablingProperty' : 'active_referenceextraction_pdb', + 'enabled' : 'false' + }, + { + 'set' : 'document_software_url', + 'jobProperty' : 'export_action_set_id_document_software_url', + 'enablingProperty' : 'active_referenceextraction_software_url', + 'enabled' : 'false' + }, + { + 'set' : 'iis-entities-software', + 'jobProperty' : 'export_action_set_id_entity_software', + 'enablingProperty' : 'active_referenceextraction_software_url', + 'enabled' : 'false' + }, + { + 'set' : 'iis-communities', + 'jobProperty' : 'export_action_set_id_document_community', + 'enablingProperty' : 'active_referenceextraction_community', + 'enabled' : 'false' + }, + { + 'set' : 'iis-referenced-patents', + 'jobProperty' : 'export_action_set_id_document_patent', + 'enablingProperty' : 'active_referenceextraction_patent', + 'enabled' : 'false' + }, + { + 'set' : 'iis-entities-patent', + 'jobProperty' : 'export_action_set_id_entity_patent', + 'enablingProperty' : 'active_referenceextraction_patent', + 'enabled' : 'false' + }, + { + 'set' : 'iis-covid-19', + 'jobProperty' : 'export_action_set_id_document_covid19', + 'enablingProperty' : 'active_referenceextraction_covid19', + 'enabled' : 'false' + } + ] + + + + + + + + prepare IIS parameters + + import_islookup_service_location + import_content_objectstores_csv + import_content_object_store_location + import_mdstore_service_location + import_dataset_mdstore_ids_csv + oozie.wf.application.path + /lib/iis/primary/snapshots/2023-05-24_dhp-schemas_2.13.1-patched + IIS + deprecated - not used + import_infospace_graph_location + + import_project_concepts_context_ids_csv + argo-france,aurora,beopen,citizen-science,clarin,covid-19,dariah,dh-ch,dth,oa-pg,ebrains,egi,egrise,elixir-gr,embrc,enermaps,epos,eu-conexus,eut,eutopia,fet-fp7,fet-h2020,forthem,gotriple,heritage-science,inspired-ris,instruct,iperionhs,knowmad,lifewatch-eric,mes,neanias-atmospheric,neanias-underwater,neanias-space,netherlands,ni,north-american-studies,openaire-infrastructure,rda,risis,rural-digital-europe,science-innovation-policy,sdsn-gr,sobigdata,tunet + import_infospace_eligible_service_collectedfrom_datasourceid + 10|openaire____::2e06c1122c7df43765fdcf91080824fa + + + + + + + IIS main + + iisMainJobV3 + + { + 'cluster' : 'cluster', + 'oozie.wf.application.path' : 'oozie.wf.application.path', + 'referenceextraction_project_fundingclass_blacklist_regex' : 'referenceextraction_project_fundingclass_blacklist_regex', + + 'active_document_affiliation' : 'active_document_affiliation', + 'active_referenceextraction_project' : 'active_referenceextraction_project', + 'active_referenceextraction_dataset' : 'active_referenceextraction_dataset', + 'active_referenceextraction_researchinitiative' : 'active_referenceextraction_researchinitiative', + 'active_documentsclassification' : 'active_documentsclassification', + 'active_documentssimilarity' : 'active_documentssimilarity', + 'active_citationmatching' : 'active_citationmatching', + 'active_citationmatching_relations' : 'active_citationmatching_relations', + 'active_referenceextraction_pdb' : 'active_referenceextraction_pdb', + 'active_referenceextraction_software_url' : 'active_referenceextraction_software_url', + 'active_referenceextraction_community' : 'active_referenceextraction_community', + 'active_referenceextraction_patent' : 'active_referenceextraction_patent', + 'active_referenceextraction_covid19' : 'active_referenceextraction_covid19', + 'active_referenceextraction_eoscservice' : 'active_referenceextraction_eoscservice', + + 'import_content_objectstores_csv' : 'import_content_objectstores_csv', + 'import_content_object_store_location' : 'import_content_object_store_location', + 'import_mdstore_service_location' : 'import_mdstore_service_location', + 'import_islookup_service_location' : 'import_islookup_service_location', + 'import_project_concepts_context_ids_csv' : 'import_project_concepts_context_ids_csv', + 'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv', + 'import_infospace_graph_location' : 'resolvedGraphPath', + 'import_infospace_eligible_service_collectedfrom_datasourceid' : 'import_infospace_eligible_service_collectedfrom_datasourceid', + + 'export_action_set_id_matched_doc_organizations' : 'export_action_set_id_matched_doc_organizations', + 'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets', + 'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects', + 'export_action_set_id_document_research_initiative' : 'export_action_set_id_document_research_initiative', + 'export_action_set_id_document_similarities_standard' : 'export_action_set_id_document_similarities_standard', + + 'export_action_set_id_document_referencedDocuments' : 'export_action_set_id_document_referencedDocuments', + 'export_action_set_id_citation_relations' : 'export_action_set_id_citation_relations', + 'export_action_set_id_document_pdb' : 'export_action_set_id_document_pdb', + 'export_action_set_id_document_software_url' : 'export_action_set_id_document_software_url', + 'export_action_set_id_entity_software' : 'export_action_set_id_entity_software', + 'export_action_set_id_document_community' : 'export_action_set_id_document_community', + 'export_action_set_id_document_patent' : 'export_action_set_id_document_patent', + 'export_action_set_id_entity_patent' : 'export_action_set_id_entity_patent', + 'export_action_set_id_document_covid19' : 'export_action_set_id_document_covid19', + 'export_action_set_id_document_classes' : 'export_action_set_id_document_classes', + 'export_action_set_id_document_eoscServices' : 'export_action_set_id_document_eoscServices' + } + + + { + 'import_content_pdfaggregation_table_name' : 'pdfaggregation_i.payload' + } + + false + build-report + + + + + + + update IIS action sets + + + + + + + create the INFERRED graph + + executeOozieJob + IIS + + { + 'inputActionSetIds' : 'actionSetIdsIISGraph', + 'inputGraphRootPath' : 'resolvedGraphPath', + 'outputGraphRootPath' : 'inferredGraphPath', + 'isLookupUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/actionmanager/wf/main/oozie_app', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '12G', + 'activePromoteDatasetActionPayload' : 'true', + 'activePromoteDatasourceActionPayload' : 'true', + 'activePromoteOrganizationActionPayload' : 'true', + 'activePromoteOtherResearchProductActionPayload' : 'true', + 'activePromoteProjectActionPayload' : 'true', + 'activePromotePublicationActionPayload' : 'true', + 'activePromoteRelationActionPayload' : 'true', + 'activePromoteResultActionPayload' : 'true', + 'activePromoteSoftwareActionPayload' : 'true', + 'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET', + 'shouldGroupById':'false', + 'workingDir' : '/tmp/prod_provision/working_dir/promoteActionsIIS' + } + + build-report + + + + + + + search for duplicates in the raw graph + + executeOozieJob + IIS + + { + 'actionSetId' : 'dedupConfig', + 'graphBasePath' : 'inferredGraphPath', + 'dedupGraphPath': 'dedupGraphPath', + 'isLookUpUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/dedup/scan/oozie_app', + 'actionSetIdOpenorgs' : 'dedup-similarity-organization-simple', + 'workingPath' : '/tmp/prod_provision/working_dir/dedup', + 'whiteListPath' : '/data/dedup/whitelist_prod', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', + 'sparkDriverMemory' : '6G' + } + + build-report + + + + + + + mark duplicates as deleted and redistribute the relationships + + executeOozieJob + IIS + + { + 'graphBasePath' : 'dedupGraphPath', + 'graphOutputPath': 'consistentGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/dedup/consistency/oozie_app', + 'workingPath' : '/tmp/prod_provision/working_dir/dedup', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '10G' + } + + build-report + + + + + + + applies the actionsets referring to deduped records + + executeOozieJob + IIS + + { + 'inputActionSetIds' : 'actionSetIdsDedup', + 'inputGraphRootPath' : 'consistentGraphPath', + 'outputGraphRootPath' : 'dedupEnrichedGraphPath', + 'isLookupUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/actionmanager/wf/main/oozie_app', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', + 'activePromoteDatasetActionPayload' : 'true', + 'activePromoteDatasourceActionPayload' : 'true', + 'activePromoteOrganizationActionPayload' : 'true', + 'activePromoteOtherResearchProductActionPayload' : 'true', + 'activePromoteProjectActionPayload' : 'true', + 'activePromotePublicationActionPayload' : 'true', + 'activePromoteRelationActionPayload' : 'true', + 'activePromoteResultActionPayload' : 'true', + 'activePromoteSoftwareActionPayload' : 'true', + 'mergeAndGetStrategy' : 'MERGE_FROM_AND_GET', + 'shouldGroupById':'true', + 'workingDir' : '/tmp/prod_provision/working_dir/promoteActionsDedup' + } + + build-report + + + + + + + Continue after dedup enriched graph? + + YES + + + + + + + + propagates ORCID among results linked by allowedsemrels semantic relationships + + executeOozieJob + IIS + + { + 'sourcePath' : 'dedupEnrichedGraphPath', + 'outputPath': 'orcidGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/orcidtoresultfromsemrel/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/orcid', + 'allowedsemrels' : 'IsSupplementedBy;IsSupplementTo;isSupplementedBy;isSupplementTo', + 'saveGraph' : 'true', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G' + } + + build-report + + + + + + + mark results respecting some rules as belonging to communities + + executeOozieJob + IIS + + { + 'sourcePath' : 'orcidGraphPath', + 'outputPath': 'bulkTaggingGraphPath', + 'isLookUpUrl' : 'isLookUpUrl', + 'pathMap' : 'bulkTaggingPathMap' + } + + + { + 'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus', + 'postgresUser' : 'dnet', + 'postgresPassword' : 'dnetPwd', + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/bulktag/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/bulktag', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '10G' + } + + build-report + + + + + + + creates relashionships between results and organizations when the organizations are associated to institutional repositories + + executeOozieJob + IIS + + { + 'sourcePath' : 'bulkTaggingGraphPath', + 'outputPath': 'affiliationGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/affiliation/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/affiliation', + 'saveGraph' : 'true', + 'blacklist' : 'empty' + } + + build-report + + + + + + + creates relashionships between results and organizations when the organizations are in parent/child relationships. The childs to the parents + + executeOozieJob + IIS + + { + 'sourcePath' : 'affiliationGraphPath', + 'outputPath': 'affiliationSemRelGraphPath', + 'iterations':'iterations' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/affiliation_semrel/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/affiliationsemrel', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '10G' + } + + build-report + + + + + + + marks as belonging to communities the result collected from datasources related to the organizations specified in the organizationCommunityMap + + executeOozieJob + IIS + + { + 'sourcePath' : 'affiliationSemRelGraphPath', + 'outputPath': 'communityOrganizationGraphPath', + 'organizationtoresultcommunitymap': 'propagationOrganizationCommunityMap' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/community_organization/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/community_organization', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '12G', + 'saveGraph' : 'true' + } + + build-report + + + + + + + created relation between projects and results linked to other results trough allowedsemrel semantic relations linked to projects + + executeOozieJob + IIS + + { + 'sourcePath' : 'communityOrganizationGraphPath', + 'outputPath': 'fundingGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/funding/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/funding', + 'allowedsemrels' : 'IsSupplementedBy;IsSupplementTo', + 'saveGraph' : 'true', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '10G' + } + + build-report + + + + + + + tag as belonging to communitites result in in allowedsemrels relation with other result already linked to communities + + executeOozieJob + IIS + + { + 'sourcePath' : 'fundingGraphPath', + 'outputPath': 'communitySemRelGraphPath', + 'isLookUpUrl' : 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/community_semrel/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/community_semrel', + 'allowedsemrels' : 'IsSupplementedBy;IsSupplementTo', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '10G', + 'saveGraph' : 'true' + } + + build-report + + + + + + + associated to results colleced from allowedtypes and those in the whithelist the country of the organization(s) handling the datasource it is collected from + + executeOozieJob + IIS + + { + 'sourcePath' : 'communitySemRelGraphPath', + 'outputPath': 'countryGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/country/oozie_app', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '10G', + 'workingDir' : '/tmp/prod_provision/working_dir/country', + 'allowedtypes' : 'pubsrepository::institutional', +'whitelist':'10|opendoar____::16e6a3326dd7d868cbc926602a61e4d0;10|openaire____::fdb035c8b3e0540a8d9a561a6c44f4de;10|eurocrisdris::fe4903425d9040f680d8610d9079ea14;10|openaire____::5b76240cc27a58c6f7ceef7d8c36660e;10|openaire____::172bbccecf8fca44ab6a6653e84cb92a;10|openaire____::149c6590f8a06b46314eed77bfca693f;10|eurocrisdris::a6026877c1a174d60f81fd71f62df1c1;10|openaire____::4692342f0992d91f9e705c26959f09e0;10|openaire____::8d529dbb05ec0284662b391789e8ae2a;10|openaire____::345c9d171ef3c5d706d08041d506428c;10|opendoar____::1c1d4df596d01da60385f0bb17a4a9e0;10|opendoar____::7a614fd06c325499f1680b9896beedeb;10|opendoar____::1ee3dfcd8a0645a25a35977997223d22;10|opendoar____::d296c101daa88a51f6ca8cfc1ac79b50;10|opendoar____::798ed7d4ee7138d49b8828958048130a;10|openaire____::c9d2209ecc4d45ba7b4ca7597acb88a2;10|eurocrisdris::c49e0fe4b9ba7b7fab717d1f0f0a674d;10|eurocrisdris::9ae43d14471c4b33661fedda6f06b539;10|eurocrisdris::432ca599953ff50cd4eeffe22faf3e48', + 'saveGraph' : 'true' + } + + build-report + + + + + + + clean the properties in the graph typed as Qualifier according to the vocabulary indicated in schemeid + + executeOozieJob + IIS + + { + 'graphInputPath' : 'countryGraphPath', + 'graphOutputPath': 'cleanedGraphPath', + 'isLookupUrl': 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/graph/clean/oozie_app', + 'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus', + 'postgresUser' : 'dnet', + 'postgresPassword' : 'dnetPwd', + 'verifyCountryParam' : '10.17632;10.5061', + 'collectedfrom' : 'NARCIS', + 'country' : 'NL', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', + 'workingDir' : '/tmp/prod_provision/working_dir/clean', + 'shouldClean' : 'true' + } + + build-report + + + + + + + removes blacklisted relations + + executeOozieJob + IIS + + { + 'sourcePath' : 'cleanedGraphPath', + 'outputPath': 'blacklistedGraphPath' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/PROD/oa/enrichment/blacklist/oozie_app', + 'workingDir' : '/tmp/prod_provision/working_dir/blacklist', + 'postgresURL' : 'jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus', + 'postgresUser' : 'dnet', + 'postgresPassword' : 'dnetPwd', + 'sparkExecutorCores' : '2', + 'sparkExecutorMemory' : '10G' + } + + build-report + + + + + + + + wf_20230605_132025_368 + 2023-06-05T13:21:02+00:00 + FAILURE + eu.dnetlib.data.hadoop.rmi.HadoopServiceException: hadoop job: 0011965-221214081318119-oozie-oozi-W failed with status: KILLED, oozie log: 2023-06-05 13:20:37,012 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] No results found 2023-06-05 13:20:37,032 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] Start action [0011965-221214081318119-oozie-oozi-W@:start:] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:37,032 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] [***0011965-221214081318119-oozie-oozi-W@:start:***]Action status=DONE 2023-06-05 13:20:37,032 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] [***0011965-221214081318119-oozie-oozi-W@:start:***]Action updated in DB! 2023-06-05 13:20:37,078 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] No results found 2023-06-05 13:20:37,093 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@:start:] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@:start: 2023-06-05 13:20:37,093 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:37,140 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] Start action [0011965-221214081318119-oozie-oozi-W@reset_outputpath] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,251 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] [***0011965-221214081318119-oozie-oozi-W@reset_outputpath***]Action status=DONE 2023-06-05 13:20:48,251 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] [***0011965-221214081318119-oozie-oozi-W@reset_outputpath***]Action updated in DB! 2023-06-05 13:20:48,299 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] No results found 2023-06-05 13:20:48,322 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] Start action [0011965-221214081318119-oozie-oozi-W@copy_entities] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,322 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] [***0011965-221214081318119-oozie-oozi-W@copy_entities***]Action status=DONE 2023-06-05 13:20:48,322 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] [***0011965-221214081318119-oozie-oozi-W@copy_entities***]Action updated in DB! 2023-06-05 13:20:48,367 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,372 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,378 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,384 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No results found 2023-06-05 13:20:48,491 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] Start action [0011965-221214081318119-oozie-oozi-W@copy_organization] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,495 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] Start action [0011965-221214081318119-oozie-oozi-W@copy_relation] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,505 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] Start action [0011965-221214081318119-oozie-oozi-W@copy_datasources] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:48,513 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] Start action [0011965-221214081318119-oozie-oozi-W@copy_projects] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2023-06-05 13:20:49,676 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] checking action, hadoop job ID [job_1676554762303_45231] status [RUNNING] 2023-06-05 13:20:49,681 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] [***0011965-221214081318119-oozie-oozi-W@copy_relation***]Action status=RUNNING 2023-06-05 13:20:49,681 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] [***0011965-221214081318119-oozie-oozi-W@copy_relation***]Action updated in DB! 2023-06-05 13:20:49,692 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_relation 2023-06-05 13:20:49,717 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] checking action, hadoop job ID [job_1676554762303_45234] status [RUNNING] 2023-06-05 13:20:49,721 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] [***0011965-221214081318119-oozie-oozi-W@copy_projects***]Action status=RUNNING 2023-06-05 13:20:49,721 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] [***0011965-221214081318119-oozie-oozi-W@copy_projects***]Action updated in DB! 2023-06-05 13:20:49,733 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_projects 2023-06-05 13:20:49,764 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] checking action, hadoop job ID [job_1676554762303_45233] status [RUNNING] 2023-06-05 13:20:49,768 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] [***0011965-221214081318119-oozie-oozi-W@copy_organization***]Action status=RUNNING 2023-06-05 13:20:49,768 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] [***0011965-221214081318119-oozie-oozi-W@copy_organization***]Action updated in DB! 2023-06-05 13:20:49,771 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] checking action, hadoop job ID [job_1676554762303_45232] status [RUNNING] 2023-06-05 13:20:49,773 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_organization 2023-06-05 13:20:49,775 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] [***0011965-221214081318119-oozie-oozi-W@copy_datasources***]Action status=RUNNING 2023-06-05 13:20:49,775 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] [***0011965-221214081318119-oozie-oozi-W@copy_datasources***]Action updated in DB! 2023-06-05 13:20:49,779 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_datasources 2023-06-05 13:20:49,784 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_entities] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_entities 2023-06-05 13:20:49,784 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@reset_outputpath] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@reset_outputpath 2023-06-05 13:20:53,821 INFO org.apache.oozie.command.wf.KillXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] STARTED WorkflowKillXCommand for jobId=0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:53,826 WARN org.apache.oozie.workflow.lite.LiteWorkflowInstance: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] workflow killed, killing [5] nodes 2023-06-05 13:20:53,866 INFO org.apache.oozie.command.wf.KillXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] ENDED WorkflowKillXCommand for jobId=0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:53,866 INFO org.apache.oozie.DagEngine: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] User claudio.atzori killed the WF job 0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:54,196 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_projects 2023-06-05 13:20:54,504 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_relation 2023-06-05 13:20:54,813 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_datasources 2023-06-05 13:20:55,126 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.production] GROUP[-] TOKEN[] APP[community_to_result_propagation] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W 2023-06-05 13:20:55,126 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0011965-221214081318119-oozie-oozi-W] ACTION[0011965-221214081318119-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0011965-221214081318119-oozie-oozi-W@copy_organization + communityOrganizationPropagation + + + \ No newline at end of file