From 34172455d1057f947d299a9d8799684fce6991db Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 24 May 2023 09:56:23 +0200 Subject: [PATCH 1/8] [BulkTag] Adding remove constraints to specify when a community must not appear in the context of a result. --- .../dhp/bulktag/community/Community.java | 9 + .../community/CommunityConfiguration.java | 15 + .../CommunityConfigurationFactory.java | 14 + .../dhp/bulktag/community/ResultTagger.java | 23 +- .../resources/eu/dnetlib/dhp/bulktag/query.xq | 3 + .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 49 +- .../tagging_conf_remove.xml | 1555 +++++++++++++++++ .../dataset_10.json | 4 +- 8 files changed, 1665 insertions(+), 7 deletions(-) create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java index c218b0c2b..b44376e22 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java @@ -15,6 +15,7 @@ public class Community implements Serializable { private List providers = new ArrayList<>(); private List zenodoCommunities = new ArrayList<>(); private SelectionConstraints constraints = new SelectionConstraints(); + private SelectionConstraints removeConstraints = new SelectionConstraints(); public String toJson() { final Gson g = new Gson(); @@ -67,4 +68,12 @@ public class Community implements Serializable { public void setConstraints(SelectionConstraints constraints) { this.constraints = constraints; } + + public SelectionConstraints getRemoveConstraints() { + return removeConstraints; + } + + public void setRemoveConstraints(SelectionConstraints removeConstraints) { + this.removeConstraints = removeConstraints; + } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java index a30e1cbbb..689628869 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java @@ -28,6 +28,8 @@ public class CommunityConfiguration implements Serializable { private Map selectionConstraintsMap = new HashMap<>(); // map eosc datasource -> communityid private Map>> eoscDatasourceMap = new HashMap<>(); + //map communityid -> remove constraints + private Map removeConstraintsMap = new HashMap<>(); public Map>> getEoscDatasourceMap() { return eoscDatasourceMap; @@ -71,6 +73,14 @@ public class CommunityConfiguration implements Serializable { this.selectionConstraintsMap = selectionConstraintsMap; } + public Map getRemoveConstraintsMap() { + return removeConstraintsMap; + } + + public void setRemoveConstraintsMap(Map removeConstraintsMap) { + this.removeConstraintsMap = removeConstraintsMap; + } + CommunityConfiguration(final Map communities) { this.communities = communities; init(); @@ -90,6 +100,9 @@ public class CommunityConfiguration implements Serializable { if (selectionConstraintsMap == null) { selectionConstraintsMap = Maps.newHashMap(); } + if (removeConstraintsMap == null) { + removeConstraintsMap = Maps.newHashMap(); + } for (Community c : getCommunities().values()) { // get subjects @@ -111,6 +124,8 @@ public class CommunityConfiguration implements Serializable { zenodocommunityMap); } selectionConstraintsMap.put(id, c.getConstraints()); + + removeConstraintsMap.put(id, c.getRemoveConstraints()); } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java index da144fb7b..695bbc363 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java @@ -86,6 +86,7 @@ public class CommunityConfigurationFactory { c.setProviders(parseDatasources(node)); c.setZenodoCommunities(parseZenodoCommunities(node)); c.setConstraints(parseConstrains(node)); + c.setRemoveConstraints(parseRemoveConstrains(node)); return c; } @@ -102,6 +103,19 @@ public class CommunityConfigurationFactory { return selectionConstraints; } + private static SelectionConstraints parseRemoveConstrains(Node node) { + Node constsNode = node.selectSingleNode("./removeConstraints"); + if (constsNode == null || StringUtils.isBlank(StringUtils.trim(constsNode.getText()))) { + return new SelectionConstraints(); + } + SelectionConstraints selectionConstraints = new Gson() + .fromJson(constsNode.getText(), SelectionConstraints.class); + + selectionConstraints.setSelection(resolver); + log.info("number of selection constraints set " + selectionConstraints.getCriteria().size()); + return selectionConstraints; + } + private static List parseSubjects(final Node node) { final List subjects = Lists.newArrayList(); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java index 3f0954f38..0be2f59a9 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java @@ -79,6 +79,23 @@ public class ResultTagger implements Serializable { break; } + // communities contains all the communities to be not added to the context + final Set removeCommunities = new HashSet<>(); + + conf + .getRemoveConstraintsMap() + .keySet() + .forEach(communityId -> { + if (conf.getRemoveConstraintsMap().get(communityId).getCriteria() != null && + conf + .getRemoveConstraintsMap() + .get(communityId) + .getCriteria() + .stream() + .anyMatch(crit -> crit.verifyCriteria(param))) + removeCommunities.add(communityId); + }); + // communities contains all the communities to be added as context for the result final Set communities = new HashSet<>(); @@ -164,7 +181,8 @@ public class ResultTagger implements Serializable { .getSelectionConstraintsMap() .keySet() .forEach(communityId -> { - if (conf.getSelectionConstraintsMap().get(communityId).getCriteria() != null && + if (!removeCommunities.contains(communityId) && + conf.getSelectionConstraintsMap().get(communityId).getCriteria() != null && conf .getSelectionConstraintsMap() .get(communityId) @@ -175,6 +193,9 @@ public class ResultTagger implements Serializable { }); communities.addAll(aconstraints); + + communities.removeAll(removeCommunities); + if (aconstraints.size() > 0) log.info("Found {} for advancedConstraints ", aconstraints.size()); diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq index a9c0d9e3f..052037efb 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq @@ -10,6 +10,9 @@ where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] and $x//con return { $x//CONFIGURATION/context/@id} + +{$x//CONFIGURATION/context/param[./@name='removeConstraints']/text() } + {$x//CONFIGURATION/context/param[./@name='advancedConstraints']/text() } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 4432210f1..aff5ee5fb 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -39,8 +39,10 @@ public class BulkTagJobTest { + " \"contributor\" : \"$['contributor'][*]['value']\"," + " \"description\" : \"$['description'][*]['value']\", " + " \"subject\" :\"$['subject'][*]['value']\" , " + - "\"fos\" : \"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"" + - "} "; + "\"fos\" : \"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"," + + "\"sdg\" : \"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"," + + "\"hostedby\" : \"$['instance'][*]['hostedby']['key']\" , " + + "\"collectedfrom\" : \"$['instance'][*]['collectedfrom']['key']\"} "; private static SparkSession spark; @@ -56,7 +58,7 @@ public class BulkTagJobTest { .toString( BulkTagJobTest.class .getResourceAsStream( - "/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_dth.xml")); + "/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml")); } catch (IOException e) { e.printStackTrace(); } @@ -1525,4 +1527,43 @@ public class BulkTagJobTest { .count()); } -} + + @Test + void removeTest() throws Exception { + final String pathMap = BulkTagJobTest.pathMap; + SparkBulkTagJob + .main( + new String[]{ + "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints").getPath(), + "-taggingConf", taggingConf, + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", + "-outputPath", workingDir.toString() + "/dataset", + "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-pathMap", pathMap + }); + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + + Assertions.assertEquals(12, tmp.count()); + org.apache.spark.sql.Dataset verificationDataset = spark + .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); + + verificationDataset.createOrReplaceTempView("dataset"); + String query = "select id, MyT.id community, MyD.provenanceaction.classid provenance, MyD.provenanceaction.classname name " + + "from dataset " + + "lateral view explode(context) c as MyT " + + "lateral view explode(MyT.datainfo) d as MyD " + + "where MyD.inferenceprovenance = 'bulktagging'"; + + org.apache.spark.sql.Dataset idExplodeCommunity = spark.sql(query); + + idExplodeCommunity.show(false); + } + + } diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml new file mode 100644 index 000000000..edd6c7e0a --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml @@ -0,0 +1,1555 @@ + + + + + + + zenodo + + + + + + + + + + + + + + + + + + + + + + re3data_____::a507cdacc5bbcc08761c92185dee5cab + + + + + + + + + + + + rda + + + + + + + + SDG13 - Climate action + SDG8 - Decent work and economic growth + SDG15 - Life on land + SDG2 - Zero hunger + SDG17 - Partnerships for the goals + SDG10 - Reduced inequalities + SDG5 - Gender equality + SDG12 - Responsible consumption and production + SDG14 - Life below water + SDG6 - Clean water and sanitation + SDG11 - Sustainable cities and communities + SDG1 - No poverty + SDG3 - Good health and well being + SDG7 - Affordable and clean energy + SDG4 - Quality education + SDG9 - Industry innovation and infrastructure + SDG16 - Peace justice and strong institutions + + + + + + + + modern art + monuments + europeana data model + sites + field walking + frescoes + LIDO metadata schema + art history + excavation + Arts and Humanities General + cities + coins + temples + numismatics + lithics + roads + environmental archaeology + digital cultural heritage + archaeological reports + history + CRMba + churches + cultural heritage + archaeological stratigraphy + religious art + buidings + digital humanities + survey + archaeological sites + linguistic studies + bioarchaeology + architectural orders + palaeoanthropology + fine arts + europeana + CIDOC CRM + decorations + classic art + stratigraphy + digital archaeology + intangible cultural heritage + walls + humanities + chapels + CRMtex + Language and Literature + paintings + archaeology + fair data + mosaics + burials + architecture + medieval art + castles + CARARE metadata schema + statues + natural language processing + inscriptions + CRMsci + vaults + contemporary art + Arts and Humanities + CRMarchaeo + pottery + site + architectural + vessels + + + + re3data_____::9ebe127e5f3a0bf401875690f3bb6b81 + + + + doajarticles::c6cd4b532e12868c1d760a8d7cda6815 + + + + doajarticles::a6de4499bb87bf3c01add0a9e2c9ed0b + + + + doajarticles::6eb31d13b12bc06bbac06aef63cf33c9 + + + + doajarticles::0da84e9dfdc8419576169e027baa8028 + + + + re3data_____::84e123776089ce3c7a33db98d9cd15a8 + + + + openaire____::c5502a43e76feab55dd00cf50f519125 + + + + re3data_____::a48f09c562b247a9919acfe195549b47 + + + + opendoar____::97275a23ca44226c9964043c8462be96 + + + + + + storm + + + + crosscult + + + + wholodance_eu + + + + digcur2013 + + + + gravitate + + + + dipp2014 + + + + digitalhumanities + + + + dimpo + + + + adho + + + + chc + + + + wahr + + + + ibe + + + + ariadne + + + + parthenos-hub + + + + parthenos-training + + + + gandhara + + + + cmsouthasia + + + + nilgirihills + + + + shamsa_mustecio + + + + bodhgaya + + + + oac_dh-ch + + + + + + + Stock Assessment + pelagic + Acoustic + Fish farming + Fisheries + Fishermen + maximum sustainable yield + trawler + Fishing vessel + Fisherman + Fishing gear + mackerel + RFMO + Fish Aggregating Device + Bycatch + Fishery + common fisheries policy + Fishing fleet + Aquaculture + + + + doajarticles::8cec81178926caaca531afbd8eb5d64c + + + + doajarticles::0f7a7f30b5400615cae1829f3e743982 + + + + doajarticles::9740f7f5af3e506d2ad2c215cdccd51a + + + + doajarticles::9f3fbaae044fa33cb7069b72935a3254 + + + + doajarticles::cb67f33eb9819f5c624ce0313957f6b3 + + + + doajarticles::e21c97cbb7a209afc75703681c462906 + + + + doajarticles::554cde3be9e5c4588b4c4f9f503120cb + + + + tubitakulakb::11e22f49e65b9fd11d5b144b93861a1b + + + + doajarticles::57c5d3837da943e93b28ec4db82ec7a5 + + + + doajarticles::a186f5ddb8e8c7ecc992ef51cf3315b1 + + + + doajarticles::e21c97cbb7a209afc75703681c462906 + + + + doajarticles::dca64612dfe0963fffc119098a319957 + + + + doajarticles::dd70e44479f0ade25aa106aef3e87a0a + + + + + + discardless + + + + farfish2020 + + + + facts + + + + climefish + + + + proeel + + + + primefish + + + + h2020_vicinaqua + + + + meece + + + + rlsadb + + + + iotc_ctoi + + + + + + + + brain mapping + brain imaging + electroencephalography + arterial spin labelling + brain fingerprinting + brain + neuroimaging + Multimodal Brain Image Analysis + fMRI + neuroinformatics + fetal brain + brain ultrasonic imaging + topographic brain mapping + diffusion tensor imaging + computerized knowledge assessment + connectome mapping + brain magnetic resonance imaging + brain abnormalities + + + + re3data_____::5b9bf9171d92df854cf3c520692e9122 + + + + doajarticles::c7d3de67dc77af72f6747157441252ec + + + + re3data_____::8515794670370f49c1d176c399c714f5 + + + + doajarticles::d640648c84b10d425f96f11c3de468f3 + + + + doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a + + + + rest________::fb1a3d4523c95e63496e3bc7ba36244b + + + + + + neuroinformatics + + + + hbp + + + + from_neuroscience_to_machine_learning + + + + ci2c + + + + opensourcebrain + + + + brainspeak + + + + braincom + + + + nextgenvis + + + + meso-brain + + + + neuroplasticity-workshop + + + + bionics + + + + brainmattrain-676408 + + + + repronim + + + + affectiveneuro + + + + con + + + + lab_neurol_sperim_irfmn_irccs_milano_it + + + + + + + + marine + ocean + fish + aqua + sea + + + + + adriplan + + + + devotes-project + + + + euro-basin + + + + naclim + + + + discardless + + + + assisibf + + + + meece + + + + facts + + + + proeel + + + + aquatrace + + + + myfish + + + + atlas + + + + blue-actionh2020 + + + + sponges + + + + merces_project + + + + bigdataocean + + + + columbus + + + + h2020-aquainvad-ed + + + + aquarius + + + + southern-ocean-observing-system + + + + eawag + + + + mossco + + + + onc + + + + oceanbiogeochemistry + + + + oceanliteracy + + + + openearth + + + + ocean + + + + calcifierraman + + + + bermudabream + + + + brcorp1 + + + + mce + + + + biogeochem + + + + ecc2014 + + + + fisheries + + + + sedinstcjfas + + + + narmada + + + + umr-entropie + + + + farfish2020 + + + + primefish + + + + zf-ilcs + + + + climefish + + + + afrimed_eu + + + + spi-ace + + + + cice-consortium + + + + nemo-ocean + + + + mesopp-h2020 + + + + marxiv + + + + + + + + + + + instruct + + + + west-life + + + + + + + + + + + + + + animal production and health + fisheries and aquaculture + food safety and human nutrition + information management + food technology + agri-food education and extension + natural resources and environment + food system + engineering technology and Research + agriculture + food safety risk assessment + food security + farming practices and systems + plant production and protection + agri-food economics and policy + Agri-food + food distribution + forestry + + + + opendoar____::1a551829d50f1400b0dab21fdd969c04 + + + + opendoar____::49af6c4e558a7569d80eee2e035e2bd7 + + + + opendoar____::0266e33d3f546cb5436a10798e657d97 + + + + opendoar____::fd4c2dc64ccb8496e6f1f94c85f30d06 + + + + opendoar____::41bfd20a38bb1b0bec75acf0845530a7 + + + + opendoar____::87ae6fb631f7c8a627e8e28785d9992d + + + + + + edenis + + + + efsa-pilot + + + + egene3 + + + + efsa-kj + + + + euromixproject + + + + discardless + + + + sedinstcjfst + + + + afinet-kc + + + + 2231-4784 + + + + 2231-0606 + + + + solace + + + + pa17 + + + + smartakis + + + + sedinstcjae + + + + phenology_camera + + + + aginfra + + + + erosa + + + + bigdatagrapes + + + + + + + {"criteria":[ + {"constraint":[{"verb":"equals","field":"hostedby","value":"opendoar____::fake"}]}, + {"constraint":[ + {"verb":"equals","field":"collectedfrom","value":"opendoar____::fake"}]} + ]} + + {"criteria":[ + {"constraint":[ + {"verb":"equals_caseinsensitive","field":"subject","value":"digital twins"}, + {"verb":"contains_caseinsensitive","field":"subject","value":"health"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"structural"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"marine"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"avionics"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"battery"} + ]}, + {"constraint":[ + {"verb":"contains_caseinsensitive","field":"title","value":"Human Digital Twins"} + ]}, + {"constraint":[ + {"verb":"contains_caseinsensitive","field":"description","value":"Human Digital Twins"} + ]}, + {"constraint":[ + {"verb":"equals_caseinsensitive","field":"subject","value":"Human Digital Twins"} + ]}, + {"constraint":[ + {"verb":"contains_caseinsensitive","field":"title","value":"Virtual Human Twin"} + ]}, + {"constraint":[ + {"verb":"contains_caseinsensitive","field":"description","value":"Virtual Human Twin"} + ]}, + {"constraint":[ + {"verb":"equals_caseinsensitive","field":"subject","value":"Virtual Human Twin"} + ]}, + {"constraint":[ + {"verb":"equals_caseinsensitive","field":"subject","value":"digital twin"}, + {"verb":"contains_caseinsensitive","field":"subject","value":"health"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"structural"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"marine"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"avionics"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"battery"} + ]}, + {"constraint":[ + {"verb":"contains_caseinsensitive","field":"title","value":"digital twin health"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"Acoustic"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"Health Monitoring"}, + {"verb":"not_contains_caseinsensitive","field":"title","value":"Health Monitoring"}, + {"verb":"not_contains_caseinsensitive","field":"title","value":"Health Management"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"Health Assessment"}, + {"verb":"not_contains_caseinsensitive","field":"title","value":"Health Assessment"}, + {"verb":"not_contains_caseinsensitive","field":"title","value":"Health status"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"ELECTRICAL ENGINEERING"}, + {"verb":"not_contains_caseinsensitive","field":"subject","value":"Control and Systems Engineering"} + ]} + ]} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + opendoar____::7e7757b1e12abcb736ab9a754ffb617a + {"criteria":[{"constraint":[{"verb":"contains","field":"contributor","value":"DARIAH"}]}]} + + + opendoar____::96da2f590cd7246bbde0051047b0d6f7 + {"criteria":[{"constraint":[{"verb":"contains","field":"contributor","value":"DARIAH"}]}]} + + + + + dimpo + + + + + + + + + + + + + + + + + + + + Green Transport + City mobility systems + Vulnerable road users + Traffic engineering + Transport electrification + Mobility + Intermodal freight transport + Clean vehicle fleets + Intelligent mobility + Inflight refueling + District mobility systems + Navigation and control systems for optimised planning and routing + European Space Technology Platform + European Transport networks + Green cars + Inter-modality infrastructures + Advanced Take Off and Landing Ideas + Sustainable urban systems + port-area railway networks + Innovative forms of urban transport + Alliance for Logistics Innovation through Collaboration in Europe + Advisory Council for Aeronautics Research in Europe + Mobility services for people and goods + Guidance and traffic management + Passenger mobility + Smart mobility and services + transport innovation + high-speed railway + Vehicle design + Inland shipping + public transportation + aviation’s climate impact + Road transport + On-demand public transport + Personal Air Transport + Transport + transport vulnerability + Pipeline transport + European Association of Aviation Training and Education Organisations + Defrosting of railway infrastructure + Inclusive and affordable transport + River Information Services + jel:L92 + Increased use of public transport + Seamless mobility + STRIA + trolleybus transport + Intelligent Transport System + Low-emission alternative energy for transport + Shared mobility for people and goods + Business model for urban mobility + Interoperability of transport systems + Cross-border train slot booking + Air transport + Transport pricing + Sustainable transport + European Rail Transport Research Advisory Council + Alternative aircraft configurations + Transport and Mobility + Railways applications + urban transport + Environmental impact of transport + urban freight delivery systems + Automated Road Transport + Alternative fuels in public transport + Active LIDAR-sensor for GHG-measurements + Autonomous logistics operations + Rational use of motorised transport + Network and traffic management systems + electrification of railway wagons + Single European Sky + Electrified road systems + transportation planning + Railway dynamics + Motorway of the Sea + smart railway communications + Maritime transport + Environmental- friendly transport + Combined transport + Connected automated driving technology + Innovative freight logistics services + automated and shared vehicles + Alternative Aircraft Systems + Land-use and transport interaction + Public transport system + Business plan for shared mobility + Shared mobility + Growing of mobility demand + European Road Transport Research Advisory Council + WATERBORNE ETP + Effective transport management system + Short Sea Shipping + air traffic management + Sea hubs and the motorways of the sea + Urban mobility solutions + Smart city planning + Maritime spatial planning + EUropean rail Research Network of Excellence + Transport governance + ENERGY CONSUMPTION BY THE TRANSPORT SECTOR + Integrated urban plan + inland waterway services + European Conference of Transport Research Institutes + air vehicles + E-freight + Automated Driving + Automated ships + pricing for cross-border passenger transport + Vehicle efficiency + Railway transport + Electric vehicles + Road traffic monitoring + Deep sea shipping + Circular economy in transport + Traffic congestion + air transport system + Urban logistics + Rail transport + OpenStreetMap + high speed rail + Transportation engineering + Intermodal travel information + Flight Data Recorders + Advanced driver assistance systems + long distance freight transport + Inland waterway transport + Smart mobility + Mobility integration + Personal Rapid Transit system + Safety measures & requirements for roads + Green rail transport + Electrical + Vehicle manufacturing + Future Airport Layout + Rail technologies + European Intermodal Research Advisory Council + inland navigation + Automated urban vehicles + ECSS-standards + Traveller services + Polluting transport + Air Traffic Control + Cooperative and connected and automated transport + Innovative powertrains + Quality of transport system and services + door-to- door logistics chain + Inter-modal aspects of urban mobility + travel (and mobility) + Innovative freight delivery systems + urban freight delivery infrastructures + + + + doajarticles::1c5bdf8fca58937894ad1441cca99b76 + + + + doajarticles::b37a634324a45c821687e6e80e6f53b4 + + + + doajarticles::4bf64f2a104040e4e055cd9594b2d77c + + + + doajarticles::479ca537c12755d1868bbf02938a900c + + + + doajarticles::55f31df96a60e2309f45b7c265fcf7a2 + + + + doajarticles::c52a09891a5301f9986ebbfe3761810c + + + + doajarticles::379807bc7f6c71a227ef1651462c414c + + + + doajarticles::36069db531a00b85a2e8fb301f4bdc19 + + + + doajarticles::b6a898da311ded96fabf49c520b80d5d + + + + doajarticles::d0753d9180b35a271d8b4a31f449749f + + + + doajarticles::172050a92511838393a3fe237ae47e31 + + + + doajarticles::301ed96c62abb160a3e29796efe5c95c + + + + doajarticles::0f4f805b3d842f2c7f1b077c3426fa59 + + + + doajarticles::ba73728b84437b8d48ae287b867c7215 + + + + doajarticles::86faef424d804309ccf45f692523aa48 + + + + doajarticles::73bd758fa41671de70964c3ecba013af + + + + doajarticles::e661fc0bdb24af42b740a08f0ddc6cf4 + + + + doajarticles::a6d3052047d5dbfbd43d95b4afb0f3d7 + + + + doajarticles::ca61df07089acc53a1569bde6673d82a + + + + doajarticles::237dd6f1606600459d0297abd8ed9976 + + + + doajarticles::fba6191177ede7c51ea1cdf58eae7f8b + + + + + + jsdtl + + + + utc-martrec + + + + utc-uti + + + + stp + + + + c2smart + + + + stride-utc + + + + crowd4roads + + + + lemo + + + + imov3d + + + + tra2018 + + + + optimum + + + + stars + + + + iecteim + + + + iccpt2019 + + + + + + + + + + + + + + + + + + + + + + + + + + + {"criteria":[{"constraint":[{"verb":"equals_caseinsensitive","field":"subject","value":"ciencias de la comunicación"}, + {"verb":"equals","field":"subject","value":"Miriam"}]}, + {"constraint":[{"verb":"equals","field":"subject","value":"miriam"}]}]} + + Sustainability-oriented science policy + STI policies + science—society relations + Science & Technology Policy + Innovation policy + science policy + Policy and Law + + + + doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d + + + + doajarticles::ae4c7286c79590f19fdca670156ce816 + + + + doajarticles::0f664bce92ce953e0c7a92068c46bfb3 + + + + doajarticles::00017183dc4c858fb77541985323a4ef + + + + doajarticles::93b306f458cce3d7aaaf58c0a725f4f9 + + + + doajarticles::9dbf8fbf3e9fe0fe1fc01e55fbd90bfc + + + + doajarticles::a2bda8785c863279bba4b8f34827b4c9 + + + + doajarticles::019a1fcb42c3fea1c1b689df76330b58 + + + + doajarticles::0daa8281938831e9c82bfed8b55a2975 + + + + doajarticles::f67ad6d268162079b3abd51a24468744 + + + + doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d + + + + doajarticles::ad114356e196a4a3d84dda59c720dacd + + + + doajarticles::01e8a54fdecaaf354c67a2dd74ae7d4f + + + + doajarticles::449305f096b10a9464449ff2d0e10e06 + + + + doajarticles::982c0c0ac378256254cce2fa6572bb6c + + + + doajarticles::49d6ed47138884566ce93cf0ccb12c02 + + + + doajarticles::a98e820dbc2e8ee0fc84ab66f263267c + + + + doajarticles::50b1ce37427b36368f8f0f1317e47f83 + + + + doajarticles::f0ec29b7450b2ac5d0ad45327eeb531a + + + + doajarticles::d8d421d3b0349a7aaa93758b27a54e84 + + + + doajarticles::7ffc35ac5133da01d421ccf8af5b70bc + + + + + + risis + + + + + + + + COVID-19 + Severe acute respiratory syndrome coronavirus 2 + SARS-CoV-2 + COVID19 + 2019 novel coronavirus + coronavirus disease 2019 + HCoV-19 + mesh:C000657245 + 2019-nCoV + coronavirus disease-19 + mesh:COVID-19 + COVID2019 + + + + opendoar____::358aee4cc897452c00244351e4d91f69 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}}]} + + + + re3data_____::7b0ad08687b2c960d5aeef06f811d5e6 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + driver______::bee53aa31dc2cbb538c10c2b65fa5824 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + openaire____::437f4b072b1aa198adcbc35910ff3b98 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + openaire____::081b82f96300b6a6e3d282bad31cb6e2 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + openaire____::9e3be59865b2c1c335d32dae2fe7b254 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + opendoar____::8b6dd7db9af49e67306feb59a8bdc52c + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + share_______::4719356ec8d7d55d3feb384ce879ad6c + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + share_______::bbd802baad85d1fd440f32a7a3a2c2b1 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + opendoar____::6f4922f45568161a8cdf4ad2299f6d23 + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, + {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} + + + + re3data_____::7980778c78fb4cf0fab13ce2159030dc + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCov"}]}]} + + + re3data_____::978378def740bbf2bfb420de868c460b + {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCov"}]}]} + + + + + chicago-covid-19 + + + + covid-19-senacyt-panama-sample + + + + covid-19-tx-rct-stats-review + + + + covid_19_senacyt_abc_panama + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset_10.json b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset_10.json index c9fd789fa..43a296f50 100644 --- a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset_10.json +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset_10.json @@ -1,6 +1,6 @@ {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868909,"id":"50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529","originalId":["od______3989::02dd5d2c222191b0b9bd4f33c8e96529"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[{"value":"10.4185/RLCS-2018-1243","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2020-03-03T13:05:26.091Z","dateoftransformation":"2020-03-03T13:06:53.161Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.091Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/562","datestamp":"2018-01-23T15:06:07Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Gallardo-Camacho, Jorge","name":"Jorge","surname":"Gallardo-Camacho","rank":1,"pid":null,"affiliation":null},{"fullname":"Trujillo Fernández, José Ramón","name":"José Ramón","surname":"Trujillo Fernández","rank":2,"pid":null,"affiliation":null},{"fullname":"Jorge Alonso, Ana","name":"Ana","surname":"Jorge Alonso","rank":3,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Miriam","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"viralidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"vídeo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"propaganda","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"jihadism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"virality","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"video","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5903 Ideologías Políticas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6310.13 Terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"El individualismo SARS-CoV-2 como estrategia","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Individualism as an improvised strategy","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Se analiza como el Yihadismo distribuye su mensaje propaganístico en You Tube.\nUniversidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"MediaLab - UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/562"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868954,"id":"50|od______3989::05d8c751462f9bb8d2b06956dfbc5c7b","originalId":["od______3989::05d8c751462f9bb8d2b06956dfbc5c7b"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.079Z","dateoftransformation":"2020-03-03T13:06:53.239Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.079Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/227","datestamp":"2017-11-15T16:29:55Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Rico Pérez, Marta","name":"Marta","surname":"Rico Pérez","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Eventos deportivos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Retos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"I Congreso Nuevos Retos covid en los Eventos Deportivos","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos, celebrado en la Universidad Camilo José Cela.\n\n\n\nDerivado del proyecto de investigación \"La situación de los eventos deportivos en España a través de su estructuras y de sus herramientas\", tiene como finalidad conocer la situación de los eventos deportivos en España. Se realizó un estudio descriptivo holístico en el que se investigaron elementos culturales, el perfil de los recursos humanos, la estructura de los comités, entre otras.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|re3data_____::7b0ad08687b2c960d5aeef06f811d5e6","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/227"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} -{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869097,"id":"50|od______3989::0f89464c4ac4c398fe0c71433b175a62","originalId":["od______3989::0f89464c4ac4c398fe0c71433b175a62"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.069Z","dateoftransformation":"2020-03-03T13:06:53.54Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.069Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/802","datestamp":"2018-11-14T15:30:47Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Actividades deportivas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Impacto físico","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigaciones en Ciencias de sars-cov-2 la Actividad Física y del Deporte CCAFD - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Blanca Romero, investigadora y docente de Ciencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela, describe dos de las líneas de investigación que se han desarrollado en el departamento CCAFD de UCJC: el impacto físico de las diferentes actividades deportivas y los métodos de recuperación más eficaces tras el ejercicio físico.\nBlanca también destaca la importancia en el ámbito de la salud los estudios en Ciencias de la Actividad Física y del Deporte.\nCiencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, DARIAH Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|re3data_____::7b0ad08687b2c960d5aeef06f811d5e6","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/802"],"distributionlocation":"","collectedfrom":{"key":"10|opendoar____::7e7757b1e12abcb736ab9a754ffb617a","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869097,"id":"50|od______3989::0f89464c4ac4c398fe0c71433b175a62","originalId":["od______3989::0f89464c4ac4c398fe0c71433b175a62"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.069Z","dateoftransformation":"2020-03-03T13:06:53.54Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.069Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/802","datestamp":"2018-11-14T15:30:47Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Actividades deportivas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Impacto físico","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigaciones en Ciencias de sars-cov-2 la Actividad Física y del Deporte CCAFD - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Blanca Romero, investigadora y docente de Ciencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela, describe dos de las líneas de investigación que se han desarrollado en el departamento CCAFD de UCJC: el impacto físico de las diferentes actividades deportivas y los métodos de recuperación más eficaces tras el ejercicio físico.\nBlanca también destaca la importancia en el ámbito de la salud los estudios en Ciencias de la Actividad Física y del Deporte.\nCiencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, DARIAH Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|re3data_____::7b0ad08687b2c960d5aeef06f811abcd","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/802"],"distributionlocation":"","collectedfrom":{"key":"10|opendoar____::7e7757b1e12abcb736ab9a754ffb617a","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869215,"id":"50|od______3989::1729c3988199b95d1d566851af7d3c55","originalId":["od______3989::1729c3988199b95d1d566851af7d3c55"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.653Z","dateoftransformation":"2020-03-03T13:06:53.764Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.653Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/801","datestamp":"2018-11-14T15:19:38Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Prevención","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Hidratación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportistas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Electrolitos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportes de resistencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"La importancia de la hidratación en competiciones resistencia, por Juan del Coso","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Juan del Coso, investigador y profesor en el Grado en Ciencias de la Actividad Física y del Deporte UCJC, presenta el proyecto de investigación que ha realizado el departamento CCAFD de UCJC: la prevención de desequilibrios de agua y electrolitos que se dan en los deportes de resistencia para que los deportistas realicen su actividad de una manera más segura.\nMás información: http://www.ucjc.edu/2016/03/investiga...\nUniversidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Grado en Ciencias de la Actividad Física y del Deporte UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/801"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869487,"id":"50|od______3989::2e3f34ce90520fae350a7e1148d7dcea","originalId":["od______3989::2e3f34ce90520fae350a7e1148d7dcea"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:54.218Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/800","datestamp":"2018-11-14T13:43:21Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Cafeína","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Efectos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alto rendimiento","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Competiciones","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Beneficios","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Efectos positivos y negativos del uso de la cafeína en el deporte","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) sobre la cafeína y sus efectos cuando se realiza deporte. https://www.ucjc.edu/2016/08/la-bbc-s...\n\nJuan Del Coso, director del Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela, nos describe cómo está siendo la investigación que desarrolla con deportistas de alto rendimiento y el consumo de cafeína.\n\n\"Nuestra investigación se enmarca dentro del campo de la fisiología del ejercicio y el objetivo que tenemos es investigar los efectos que tiene la cafeína en los deportistas, principalmente saber si es una ayuda ergogénica en la mayor parte de los deportes, pero también conocer qué perjuicios existen para aquellos deportistas que la utilizan para incrementar el rendimiento deportivo\", explica Del Coso.\n\nY es que los datos son muy llamativos: tres de cada cuatro deportistas de alto rendimiento utilizan la cafeína antes de la competición y muchas veces se utiliza sin saber los perjuicios que puede tener para el deportista. \"Lo que estamos investigando es cómo una sustancia, que te puede hacer rendir mejor, cómo puede afectar negativamente cuando termina una competición\".\n\nPero esta investigación no sólo puede extraer conclusiones interesantes y útiles para los deportistas profesionales, \"investigando el deporte de alto rendimiento podemos conocer los beneficios y perjuicios de la cafeína que también afectarían al deportista amateur y hacer un deporte en general más seguro\".\n\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/800"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869498,"id":"50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf","originalId":["od______3989::2f4f3c820c450bd08dac08d07cc82dcf"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.806Z","dateoftransformation":"2020-03-03T13:06:54.229Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.806Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/797","datestamp":"2018-11-14T12:12:23Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Cuesta Cano, Laura","name":"Laura","surname":"Cuesta Cano","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marketing digital","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Medios sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marcas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Empresas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Emprendedores","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"II Liga de Debate CICAE - UCJC","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6114.13 Marketing","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Laura Cuesta - Liga de Debate CICAE - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Laura Cuesta, profesora en la Universidad Camilo José Cela (UCJC) y Especialista en Marketing Digital y Medios Sociales para marcas y emprendedores. \n\nEn esta ocasión, Laura Cuesta, nos explica el significado de redes sociales y nos habla sobre el uso que le dan los usuarios.\n\nII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la segunda edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobre todo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/797"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} @@ -8,5 +8,5 @@ {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870750,"id":"50|od______3989::7fcbe3a03280663cddebfd3cb9203177","originalId":["od______3989::7fcbe3a03280663cddebfd3cb9203177"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.652Z","dateoftransformation":"2020-03-03T13:06:55.95Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.652Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/796","datestamp":"2018-11-05T13:36:54Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Redes sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"III Liga de Debate CICAE","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Instantaneidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información periodística","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Fake news","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Jorge Gallardo - Liga de Debate CICAE","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Jorge Gallardo, profesor de Derecho en la Universidad Camilo José Cela (UCJC), Doctor en Comunicación Audiovisual y MBA en Empresas Audiovisuales. Subdirector de Espejo Público en Antena 3 Noticias. \n\nEn esta ocasión, Jorge Gallardo habla sobre las redes sociales, cómo han transformado la manera en la que nos comunicamos y nos informamos. \n\nIII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la tercera edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobre todo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/796"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872001,"id":"50|od______3989::d791339867bec6d3eb2104deeb4e4961","originalId":["od______3989::d791339867bec6d3eb2104deeb4e4961"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.586Z","dateoftransformation":"2020-03-03T13:06:57.721Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.586Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/645","datestamp":"2018-04-12T07:59:27Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"North America","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"0501 fake fos","qualifier":{"classid":"subject:fos","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Tecnología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"UCJC Open Science Day 2018","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"El UCJC Open Science Day tiene como objetivo mostrar las actividades de investigación llevadas a cabo por investigadores de la Universidad Camilo José Cela (UCJC) a todos los miembros de la UCJC.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/645"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} {"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b280343","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Miriam","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Unión Europea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} -{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b28copy","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"digital twins","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"health pippo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b28copy","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"digital twins","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"health pippo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::fake","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} {"context": [{"dataInfo": [{"provenanceaction": {"classid": "community:subject", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "bulktagging", "invisible": false, "trust": "0.8"}, {"provenanceaction": {"classid": "community:subject", "classname": "Bulktagging for Community - Subject", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "bulktagging", "invisible": false, "trust": "0.8"}], "id": "knowmad"}], "dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemeid": "dnet:dataCite_resource", "schemename": "dnet:dataCite_resource"}, "pid": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "handle", "classname": "Handle", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10576/17266"}], "contributor": [], "oaiprovenance": {"originDescription": {"metadataNamespace": "http://www.openarchives.org/OAI/2.0/oai_dc/", "harvestDate": "2022-02-17T17:49:18.019Z", "baseURL": "http%3A%2F%2Fqspace.qu.edu.qa%2Foai%2Frequest", "datestamp": "2022-02-09T18:29:09Z", "altered": true, "identifier": "oai:qspace.qu.edu.qa:10576/17266"}}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "relevantdate": [], "collectedfrom": [{"key": "10|opendoar____::4f398cb9d6bc79ae567298335b51ba8a", "value": "Qatar University Institutional Repository"}], "id": "50|od______1664::602656670df91ab0bcaa3cef70797101", "subject": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:result_subject", "schemename": "dnet:result_subject"}, "value": "algorithmic pricing"}, {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:result_subject", "schemename": "dnet:result_subject"}, "value": "artificial intelligence"}, {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:result_subject", "schemename": "dnet:result_subject"}, "value": "digital clone"}, {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:result_subject", "schemename": "dnet:result_subject"}, "value": "Digital thought clone"}, {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:result_subject", "schemename": "dnet:result_subject"}, "value": "human digital twins"}, {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:result_subject", "schemename": "dnet:result_subject"}, "value": "micro-targeting"}, {"dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_classes", "invisible": false, "trust": "0.891"}, "qualifier": {"classid": "FOS", "classname": "ACM Computing Classification System", "schemeid": "dnet:subject_classification_typologies", "schemename": "dnet:subject_classification_typologies"}, "value": "03 medical and health sciences"}], "lastupdatetimestamp": 1679915243967, "author": [{"pid": [], "fullname": "Truby, Jon", "surname": "Truby", "name": "Jon", "rank": 1}, {"pid": [], "fullname": "Brown, Rafael", "surname": "Brown", "name": "Rafael", "rank": 2}], "instance": [{"refereed": {"classid": "0000", "classname": "UNKNOWN", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels"}, "hostedby": {"key": "10|opendoar____::4f398cb9d6bc79ae567298335b51ba8a", "value": "Qatar University Institutional Repository"}, "url": ["https://hdl.handle.net/10576/17266", "https://www.scopus.com/inward/record.uri?partnerID=HzOxMe3b&scp=85096910166&origin=inward"], "pid": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "handle", "classname": "Handle", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10576/17266"}], "distributionlocation": "", "alternateIdentifier": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.1080/13600834.2020.1850174"}], "dateofacceptance": {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "2020-12-01"}, "collectedfrom": {"key": "10|opendoar____::4f398cb9d6bc79ae567298335b51ba8a", "value": "Qatar University Institutional Repository"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}], "dateofcollection": "2022-02-17T17:49:18.019Z", "fulltext": [], "dateoftransformation": "2023-02-17T10:44:18.1Z", "description": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "This article explores the virtual twin and ethical implications of big data\\u2019s pursuit of human \\u2018digital thought clones\\u2019. It identifies various types of digital clones that have been developed and demonstrates how the pursuit of more accurate personalised consumer data for micro-targeting leads to the evolution of digital thought clones. The article explains the business case for digital thought clones and how this is the commercial Holy Grail for profit-seeking big data and advertisers, who have commoditised predictions of digital behaviour data. Given big data\\u2019s industrial-scale data mining and relentless commercialisation of all types of human data, this article identifies some types of protections but argues that more jurisdictions urgently need to enact legislation similar to the General Data Protection Regulation in Europe to protect people against unscrupulous and harmful uses of their data and the unauthorised development and use of digital thought clones."}], "format": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "application/pdf"}], "measures": [], "coverage": [], "externalReference": [], "publisher": {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "Routledge"}, "eoscifguidelines": [], "language": {"classid": "eng", "classname": "English", "schemeid": "dnet:languages", "schemename": "dnet:languages"}, "resulttype": {"classid": "publication", "classname": "publication", "schemeid": "dnet:result_typologies", "schemename": "dnet:result_typologies"}, "country": [{"classid": "QA", "classname": "Qatar", "dataInfo": {"provenanceaction": {"classid": "country:instrepos", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "propagation", "invisible": false, "trust": "0.85"}, "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["50|od______1664::602656670df91ab0bcaa3cef70797101", "oai:qspace.qu.edu.qa:10576/17266"], "source": [], "dateofacceptance": {"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "value": "2020-12-01"}, "title": [{"dataInfo": {"provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": "0.9"}, "qualifier": {"classid": "main title", "classname": "main title", "schemeid": "dnet:dataCite_title", "schemename": "dnet:dataCite_title"}, "value": "Human digital thought clones: the Holy Grail of artificial intelligence for big data"}]} \ No newline at end of file From 9567c13bc37d8c32c0b8b5763206b502c12ea02c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 24 May 2023 16:20:05 +0200 Subject: [PATCH 2/8] refactoring --- .../community/CommunityConfiguration.java | 2 +- .../CommunityConfigurationFactory.java | 2 +- .../dhp/bulktag/community/ResultTagger.java | 26 +++++----- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 47 ++++++++++--------- 4 files changed, 39 insertions(+), 38 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java index 689628869..e061ccd5e 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java @@ -28,7 +28,7 @@ public class CommunityConfiguration implements Serializable { private Map selectionConstraintsMap = new HashMap<>(); // map eosc datasource -> communityid private Map>> eoscDatasourceMap = new HashMap<>(); - //map communityid -> remove constraints + // map communityid -> remove constraints private Map removeConstraintsMap = new HashMap<>(); public Map>> getEoscDatasourceMap() { diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java index 695bbc363..7b9e03ef6 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java @@ -109,7 +109,7 @@ public class CommunityConfigurationFactory { return new SelectionConstraints(); } SelectionConstraints selectionConstraints = new Gson() - .fromJson(constsNode.getText(), SelectionConstraints.class); + .fromJson(constsNode.getText(), SelectionConstraints.class); selectionConstraints.setSelection(resolver); log.info("number of selection constraints set " + selectionConstraints.getCriteria().size()); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java index 0be2f59a9..5f62c10f4 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java @@ -83,18 +83,18 @@ public class ResultTagger implements Serializable { final Set removeCommunities = new HashSet<>(); conf - .getRemoveConstraintsMap() - .keySet() - .forEach(communityId -> { - if (conf.getRemoveConstraintsMap().get(communityId).getCriteria() != null && - conf - .getRemoveConstraintsMap() - .get(communityId) - .getCriteria() - .stream() - .anyMatch(crit -> crit.verifyCriteria(param))) - removeCommunities.add(communityId); - }); + .getRemoveConstraintsMap() + .keySet() + .forEach(communityId -> { + if (conf.getRemoveConstraintsMap().get(communityId).getCriteria() != null && + conf + .getRemoveConstraintsMap() + .get(communityId) + .getCriteria() + .stream() + .anyMatch(crit -> crit.verifyCriteria(param))) + removeCommunities.add(communityId); + }); // communities contains all the communities to be added as context for the result final Set communities = new HashSet<>(); @@ -182,7 +182,7 @@ public class ResultTagger implements Serializable { .keySet() .forEach(communityId -> { if (!removeCommunities.contains(communityId) && - conf.getSelectionConstraintsMap().get(communityId).getCriteria() != null && + conf.getSelectionConstraintsMap().get(communityId).getCriteria() != null && conf .getSelectionConstraintsMap() .get(communityId) diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index aff5ee5fb..a63e62471 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -40,9 +40,9 @@ public class BulkTagJobTest { + " \"description\" : \"$['description'][*]['value']\", " + " \"subject\" :\"$['subject'][*]['value']\" , " + "\"fos\" : \"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"," + - "\"sdg\" : \"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"," + + "\"sdg\" : \"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"," + "\"hostedby\" : \"$['instance'][*]['hostedby']['key']\" , " + - "\"collectedfrom\" : \"$['instance'][*]['collectedfrom']['key']\"} "; + "\"collectedfrom\" : \"$['instance'][*]['collectedfrom']['key']\"} "; private static SparkSession spark; @@ -1527,43 +1527,44 @@ public class BulkTagJobTest { .count()); } - @Test void removeTest() throws Exception { final String pathMap = BulkTagJobTest.pathMap; SparkBulkTagJob - .main( - new String[]{ - "-isTest", Boolean.TRUE.toString(), - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints").getPath(), - "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, - "-pathMap", pathMap - }); + .main( + new String[] { + "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", + getClass() + .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints") + .getPath(), + "-taggingConf", taggingConf, + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", + "-outputPath", workingDir.toString() + "/dataset", + "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-pathMap", pathMap + }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/dataset") - .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); Assertions.assertEquals(12, tmp.count()); org.apache.spark.sql.Dataset verificationDataset = spark - .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); + .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); verificationDataset.createOrReplaceTempView("dataset"); String query = "select id, MyT.id community, MyD.provenanceaction.classid provenance, MyD.provenanceaction.classname name " - + "from dataset " - + "lateral view explode(context) c as MyT " - + "lateral view explode(MyT.datainfo) d as MyD " - + "where MyD.inferenceprovenance = 'bulktagging'"; + + "from dataset " + + "lateral view explode(context) c as MyT " + + "lateral view explode(MyT.datainfo) d as MyD " + + "where MyD.inferenceprovenance = 'bulktagging'"; org.apache.spark.sql.Dataset idExplodeCommunity = spark.sql(query); idExplodeCommunity.show(false); } - } +} From 9097e718533da9f38fda55efd56aa4b99e2d34fa Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 24 May 2023 16:30:53 +0200 Subject: [PATCH 3/8] Added assertion in test --- .../src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index a63e62471..299731520 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -1564,7 +1564,8 @@ public class BulkTagJobTest { org.apache.spark.sql.Dataset idExplodeCommunity = spark.sql(query); - idExplodeCommunity.show(false); + Assertions.assertEquals(3, idExplodeCommunity.filter("community = 'dth'").count()); + } } From e45777e7e12256669ce541ad50b6820c0d945cec Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 26 May 2023 11:33:42 +0200 Subject: [PATCH 4/8] [aggregator graph] added validation for URLs mapped from oaf:fulltext --- .../raw/AbstractMdRecordToOafMapper.java | 20 ++++++++++++------- .../dhp/oa/graph/raw/OafToOafMapper.java | 8 +++++++- .../dhp/oa/graph/raw/OdfToOafMapper.java | 7 ++++++- .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 19 ++++++++++-------- pom.xml | 2 +- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index 504c9dd4c..efb860d44 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -5,8 +5,6 @@ import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*; import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.createOpenaireId; -import java.net.MalformedURLException; -import java.net.URL; import java.util.*; import java.util.stream.Collectors; @@ -17,7 +15,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import eu.dnetlib.dhp.common.Constants; @@ -27,12 +24,13 @@ import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; -import eu.dnetlib.dhp.schema.oaf.utils.PidType; public abstract class AbstractMdRecordToOafMapper { protected final VocabularyGroup vocs; + protected static final UrlValidator URL_VALIDATOR = UrlValidator.getInstance(); + private final boolean invisible; private final boolean shouldHashId; @@ -393,7 +391,7 @@ public abstract class AbstractMdRecordToOafMapper { r.setPublisher(preparePublisher(doc, info)); r.setEmbargoenddate(prepareField(doc, "//oaf:embargoenddate", info)); r.setSource(prepareSources(doc, info)); - r.setFulltext(prepareListFields(doc, "//oaf:fulltext", info)); + r.setFulltext(prepareListURL(doc, "//oaf:fulltext", info)); r.setFormat(prepareFormats(doc, info)); r.setContributor(prepareContributors(doc, info)); r.setResourcetype(prepareResourceType(doc, info)); @@ -672,6 +670,14 @@ public abstract class AbstractMdRecordToOafMapper { qualifier(paClassId, paClassName, paSchemeId, paSchemeName), trust); } + protected List> prepareListURL(final Node node, final String xpath, final DataInfo info) { + return listFields( + info, prepareListString(node, xpath) + .stream() + .filter(URL_VALIDATOR::isValid) + .collect(Collectors.toList())); + } + protected Field prepareField(final Node node, final String xpath, final DataInfo info) { return field(node.valueOf(xpath), info); } @@ -695,13 +701,13 @@ public abstract class AbstractMdRecordToOafMapper { } protected Set validateUrl(Collection url) { - UrlValidator urlValidator = UrlValidator.getInstance(); + if (Objects.isNull(url)) { return new HashSet<>(); } return url .stream() - .filter(u -> urlValidator.isValid(u)) + .filter(URL_VALIDATOR::isValid) .collect(Collectors.toCollection(HashSet::new)); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index 30f3935f5..2271a0fff 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -140,7 +140,7 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { final List alternateIdentifier = prepareResultPids(doc, info); final List pid = IdentifierFactory.getPids(alternateIdentifier, collectedfrom); - final Set pids = pid.stream().collect(Collectors.toCollection(HashSet::new)); + final Set pids = new HashSet<>(pid); instance .setAlternateIdentifier( @@ -158,6 +158,12 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { instance .setProcessingchargecurrency(field(doc.valueOf("//oaf:processingchargeamount/@currency"), info)); + prepareListURL(doc, "//oaf:fulltext", info) + .stream() + .findFirst() + .map(Field::getValue) + .ifPresent(instance::setFulltext); + final List nodes = Lists.newArrayList(doc.selectNodes("//dc:identifier")); final List url = nodes .stream() diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index 39c77bd37..1faa2fe9b 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -144,7 +144,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { final List alternateIdentifier = prepareResultPids(doc, info); final List pid = IdentifierFactory.getPids(alternateIdentifier, collectedfrom); - final Set pids = pid.stream().collect(Collectors.toCollection(HashSet::new)); + final Set pids = new HashSet<>(pid); instance .setAlternateIdentifier( @@ -161,6 +161,11 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { instance.setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info)); instance .setProcessingchargecurrency(field(doc.valueOf("//oaf:processingchargeamount/@currency"), info)); + prepareListURL(doc, "//oaf:fulltext", info) + .stream() + .findFirst() + .map(Field::getValue) + .ifPresent(instance::setFulltext); final Set url = new HashSet<>(); for (final Object o : doc diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index a5a277470..894ed33f7 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -27,7 +27,6 @@ import eu.dnetlib.dhp.common.Constants; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; -import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.PidType; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @@ -52,7 +51,7 @@ class MappersTest { } @Test - void testPublication() throws IOException, DocumentException { + void testPublication() throws IOException { final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_record.xml"))); @@ -112,13 +111,17 @@ class MappersTest { assertNotNull(i.getAccessright()); assertEquals("OPEN", i.getAccessright().getClassid()); }); - assertEquals("0001", p.getInstance().get(0).getRefereed().getClassid()); - assertNotNull(p.getInstance().get(0).getPid()); - assertTrue(p.getInstance().get(0).getPid().isEmpty()); + final Instance instance = p.getInstance().get(0); + assertEquals("0001", instance.getRefereed().getClassid()); + assertNotNull(instance.getPid()); + assertTrue(instance.getPid().isEmpty()); - assertTrue(!p.getInstance().get(0).getAlternateIdentifier().isEmpty()); - assertEquals("doi", p.getInstance().get(0).getAlternateIdentifier().get(0).getQualifier().getClassid()); - assertEquals("10.3897/oneeco.2.e13718", p.getInstance().get(0).getAlternateIdentifier().get(0).getValue()); + assertFalse(instance.getAlternateIdentifier().isEmpty()); + assertEquals("doi", instance.getAlternateIdentifier().get(0).getQualifier().getClassid()); + assertEquals("10.3897/oneeco.2.e13718", instance.getAlternateIdentifier().get(0).getValue()); + + assertNotNull(instance.getFulltext()); + assertEquals("https://oneecosystem.pensoft.net/article/13718/", instance.getFulltext()); assertNotNull(p.getBestaccessright()); assertEquals("OPEN", p.getBestaccessright().getClassid()); diff --git a/pom.xml b/pom.xml index 8a9f1dea8..63d871039 100644 --- a/pom.xml +++ b/pom.xml @@ -807,7 +807,7 @@ 3.3.3 3.4.2 [2.12,3.0) - [3.16.0] + [3.17.1-SNAPSHOT] [4.0.3] [6.0.5] [3.1.6] From 0389b57ca7cf0ba54e8949e5cecec20c3991d8bd Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 31 May 2023 11:06:58 +0200 Subject: [PATCH 5/8] added propagation for project to organization --- .../eu/dnetlib/dhp/PropagationConstant.java | 31 +++++++++- .../Leaves.java | 2 +- .../PrepareInfo.java | 47 +++++++++++---- .../PropagationCounter.java | 2 +- .../SparkResultToOrganizationFromSemRel.java | 59 +++++++++++++------ .../StepActions.java | 30 ++++------ .../input_preparation_parameter.json | 6 ++ .../input_propagation_parameter.json | 7 ++- .../oozie_app/config-default.xml | 0 .../oozie_app/workflow.xml | 5 +- .../PrepareInfoJobTest.java | 28 ++++----- .../SparkJobTest.java | 4 +- .../StepActionsTest.java | 2 +- .../childparenttest1/relation | 0 .../childparenttest2/relation | 0 .../execstep/childParentOrg/childparent | 0 .../execstep/currentIteration/leaves | 0 .../execstep/relation | 0 .../execstep/relsforiteration1/relation | 0 .../resultOrganization/resultorganization | 0 .../resultorganizationtest/relation | 0 21 files changed, 153 insertions(+), 70 deletions(-) rename dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/Leaves.java (79%) rename dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/PrepareInfo.java (78%) rename dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/PropagationCounter.java (97%) rename dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/SparkResultToOrganizationFromSemRel.java (80%) rename dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/StepActions.java (89%) rename dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/input_preparation_parameter.json (87%) rename dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/input_propagation_parameter.json (90%) rename dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/oozie_app/config-default.xml (100%) rename dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/oozie_app/workflow.xml (96%) rename dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/PrepareInfoJobTest.java (94%) rename dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/SparkJobTest.java (98%) rename dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/StepActionsTest.java (99%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/childparenttest1/relation (100%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/childparenttest2/relation (100%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/execstep/childParentOrg/childparent (100%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/execstep/currentIteration/leaves (100%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/execstep/relation (100%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/execstep/relsforiteration1/relation (100%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/execstep/resultOrganization/resultorganization (100%) rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/{resulttoorganizationfromsemrel => entitytoorganizationfromsemrel}/resultorganizationtest/relation (100%) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index 89bdf0982..53769c9fb 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -57,7 +57,10 @@ public class PropagationConstant { public static final String PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_NAME = "Propagation of affiliation to result collected from datasources of type institutional repository"; public static final String PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID = "result:organization:semrel"; - public static final String PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME = "Propagation of affiliation to result through sematic relations"; + public static final String PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME = "Propagation of affiliation to result through semantic relations"; + + public static final String PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID = "project:organization:semrel"; + public static final String PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_NAME = "Propagation of participation to project through semantic relations"; public static final String PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_ID = "result:project:semrel"; public static final String PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_NAME = "Propagation of result to project through semantic relation"; @@ -171,6 +174,32 @@ public class PropagationConstant { return newRelations; } + public static Relation getParticipantRelation( + String source, + String target, + String rel_class) { + return getRelation(source, target , + rel_class, + ModelConstants.PROJECT_ORGANIZATION, + ModelConstants.PARTICIPATION, + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID, + PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_NAME); + } + + public static Relation getAffiliationRelation( + String source, + String target, + String rel_class) { + return getRelation(source, target , + rel_class, + ModelConstants.RESULT_ORGANIZATION, + ModelConstants.AFFILIATION, + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID, + PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME); + } + public static Relation getRelation( String source, String target, diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/Leaves.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/Leaves.java similarity index 79% rename from dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/Leaves.java rename to dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/Leaves.java index 7984721e8..e010b54c0 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/Leaves.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/Leaves.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import java.io.Serializable; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PrepareInfo.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java similarity index 78% rename from dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PrepareInfo.java rename to dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java index 23909fd9a..7ad9c4cee 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PrepareInfo.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; @@ -7,6 +7,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; import java.io.Serializable; import java.util.*; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.*; @@ -14,8 +15,6 @@ import org.apache.spark.sql.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.ObjectMapper; - import eu.dnetlib.dhp.KeyValueSet; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob; @@ -47,13 +46,20 @@ public class PrepareInfo implements Serializable { "' and datainfo.deletedbyinference = false " + "GROUP BY source"; + // associate projects to all the participant orgs + private static final String PROJECT_ORGANIZATION_QUERY = "SELECT source key, collect_set(target) as valueSet " + + "FROM relation " + + "WHERE lower(relclass) = '" + ModelConstants.IS_PARTICIPANT.toLowerCase() + + "' and datainfo.deletedbyinference = false " + + "GROUP BY source"; + public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils .toString( SparkResultToOrganizationFromIstRepoJob.class .getResourceAsStream( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_preparation_parameter.json")); + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_preparation_parameter.json")); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); @@ -74,6 +80,9 @@ public class PrepareInfo implements Serializable { final String resultOrganizationPath = parser.get("resultOrgPath"); log.info("resultOrganizationPath: {}", resultOrganizationPath); + final String projectOrgPath = parser.get("projectOrganizationPath"); + log.info("projectOrgPath: {}", projectOrgPath); + final String relationPath = parser.get("relationPath"); log.info("relationPath: {}", relationPath); @@ -89,11 +98,12 @@ public class PrepareInfo implements Serializable { childParentPath, leavesPath, resultOrganizationPath, + projectOrgPath, relationPath)); } private static void prepareInfo(SparkSession spark, String inputPath, String childParentOrganizationPath, - String currentIterationPath, String resultOrganizationPath, String relationPath) { + String currentIterationPath, String resultOrganizationPath, String resultProjectPath, String relationPath) { Dataset relation = readPath(spark, inputPath + "/relation", Relation.class); relation.createOrReplaceTempView("relation"); @@ -113,14 +123,31 @@ public class PrepareInfo implements Serializable { .option("compression", "gzip") .json(resultOrganizationPath); + spark + .sql(PROJECT_ORGANIZATION_QUERY) + .as(Encoders.bean(KeyValueSet.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(resultProjectPath); + relation - .filter( - (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && - r.getRelClass().equals(ModelConstants.HAS_AUTHOR_INSTITUTION)) - .write() + .filter( + (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && + r.getRelClass().equals(ModelConstants.HAS_AUTHOR_INSTITUTION)) + .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") - .json(relationPath); + .json(relationPath + "/result"); + + relation + .filter( + (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && + r.getRelClass().equals(ModelConstants.IS_PARTICIPANT)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(relationPath + "/project"); Dataset children = spark .sql( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PropagationCounter.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PropagationCounter.java similarity index 97% rename from dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PropagationCounter.java rename to dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PropagationCounter.java index 788eff0e3..1c408d1c3 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PropagationCounter.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PropagationCounter.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import java.io.Serializable; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java similarity index 80% rename from dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java rename to dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java index cfc69a8f0..19e55a905 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; @@ -30,7 +30,8 @@ import eu.dnetlib.dhp.schema.oaf.Relation; public class SparkResultToOrganizationFromSemRel implements Serializable { private static final Logger log = LoggerFactory.getLogger(SparkResultToOrganizationFromSemRel.class); private static final int MAX_ITERATION = 5; - public static final String NEW_RELATION_PATH = "/newRelation"; + public static final String NEW_RESULT_RELATION_PATH = "/newResultRelation"; + public static final String NEW_PROJECT_RELATION_PATH = "/newProjectRelation"; public static void main(String[] args) throws Exception { @@ -62,6 +63,9 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { final String resultOrganizationPath = parser.get("resultOrgPath"); log.info("resultOrganizationPath: {}", resultOrganizationPath); + final String projectOrganizationPath = parser.get("projectOrganizationPath"); + log.info("projectOrganizationPath: {}", projectOrganizationPath); + final String workingPath = parser.get("workingDir"); log.info("workingPath: {}", workingPath); @@ -88,6 +92,7 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { leavesPath, childParentPath, resultOrganizationPath, + projectOrganizationPath, relationPath, workingPath, outputPath, @@ -98,13 +103,14 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { String leavesPath, String childParentPath, String resultOrganizationPath, + String projectOrganizationPath, String graphPath, String workingPath, String outputPath, int iterations) { if (iterations == 1) { doPropagateOnce( - spark, leavesPath, childParentPath, resultOrganizationPath, graphPath, + spark, leavesPath, childParentPath, resultOrganizationPath, projectOrganizationPath, graphPath, workingPath, outputPath); } else { @@ -130,15 +136,22 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { } private static void doPropagateOnce(SparkSession spark, String leavesPath, String childParentPath, - String resultOrganizationPath, String graphPath, String workingPath, + String resultOrganizationPath, String projectOrganizationPath, String graphPath, String workingPath, String outputPath) { StepActions .execStep( - spark, graphPath, workingPath + NEW_RELATION_PATH, + spark, graphPath + "/result", workingPath + NEW_RESULT_RELATION_PATH, leavesPath, childParentPath, resultOrganizationPath); - addNewRelations(spark, workingPath + NEW_RELATION_PATH, outputPath); + addNewRelations(spark, workingPath + NEW_RESULT_RELATION_PATH, outputPath); + + StepActions + .execStep( + spark, graphPath + "/project", workingPath + NEW_PROJECT_RELATION_PATH, + leavesPath, childParentPath, projectOrganizationPath); + + addNewRelations(spark, workingPath + NEW_PROJECT_RELATION_PATH, outputPath); } private static void doPropagate(SparkSession spark, String leavesPath, String childParentPath, @@ -151,11 +164,11 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { iteration++; StepActions .execStep( - spark, graphPath, workingPath + NEW_RELATION_PATH, + spark, graphPath, workingPath + NEW_RESULT_RELATION_PATH, leavesPath, childParentPath, resultOrganizationPath); StepActions .prepareForNextStep( - spark, workingPath + NEW_RELATION_PATH, resultOrganizationPath, leavesPath, + spark, workingPath + NEW_RESULT_RELATION_PATH, resultOrganizationPath, leavesPath, childParentPath, workingPath + "/leaves", workingPath + "/resOrg"); moveOutput(spark, workingPath, leavesPath, resultOrganizationPath); leavesCount = readPath(spark, leavesPath, Leaves.class).count(); @@ -185,7 +198,7 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { propagationCounter.getNotReachedFirstParent().add(1); } - addNewRelations(spark, workingPath + NEW_RELATION_PATH, outputPath); + addNewRelations(spark, workingPath + NEW_RESULT_RELATION_PATH, outputPath); } private static void moveOutput(SparkSession spark, String workingPath, String leavesPath, @@ -212,16 +225,24 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { .mapGroups( (MapGroupsFunction) (k, it) -> it.next(), Encoders.bean(Relation.class)) .flatMap( - (FlatMapFunction) r -> Arrays - .asList( - r, getRelation( - r.getTarget(), r.getSource(), ModelConstants.IS_AUTHOR_INSTITUTION_OF, - ModelConstants.RESULT_ORGANIZATION, - ModelConstants.AFFILIATION, - PROPAGATION_DATA_INFO_TYPE, - PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID, - PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME)) - .iterator() + (FlatMapFunction) r -> + { + if(r.getSource().startsWith("50|")){ + return Arrays + .asList( + r, getAffiliationRelation( + r.getTarget(), r.getSource(), ModelConstants.IS_AUTHOR_INSTITUTION_OF)) + .iterator(); + }else{ + return Arrays + .asList( + r, getParticipantRelation( + r.getTarget(), r.getSource(), ModelConstants.HAS_PARTICIPANT)) + .iterator(); + } + } + + , Encoders.bean(Relation.class)) .write() diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/StepActions.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java similarity index 89% rename from dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/StepActions.java rename to dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java index 1adbbe60e..5b6c397cf 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/StepActions.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.PropagationConstant.readPath; @@ -14,8 +14,6 @@ import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; @@ -28,13 +26,13 @@ public class StepActions implements Serializable { public static void execStep(SparkSession spark, String graphPath, String newRelationPath, - String leavesPath, String chldParentOrgPath, String resultOrgPath) { + String leavesPath, String chldParentOrgPath, String entityOrgPath) { Dataset relationGraph = readPath(spark, graphPath, Relation.class); // select only the relation source target among those proposed by propagation that are not already existent getNewRels( newRelationPath, relationGraph, - getPropagationRelation(spark, leavesPath, chldParentOrgPath, resultOrgPath)); + getPropagationRelation(spark, leavesPath, chldParentOrgPath, entityOrgPath, ModelConstants.HAS_AUTHOR_INSTITUTION)); } @@ -152,19 +150,20 @@ public class StepActions implements Serializable { private static Dataset getPropagationRelation(SparkSession spark, String leavesPath, String chldParentOrgPath, - String resultOrgPath) { + String entityOrgPath, + String semantics) { Dataset childParent = readPath(spark, chldParentOrgPath, KeyValueSet.class); - Dataset resultOrg = readPath(spark, resultOrgPath, KeyValueSet.class); + Dataset entityOrg = readPath(spark, entityOrgPath, KeyValueSet.class); Dataset leaves = readPath(spark, leavesPath, Leaves.class); childParent.createOrReplaceTempView("childParent"); - resultOrg.createOrReplaceTempView("resultOrg"); + entityOrg.createOrReplaceTempView("entityOrg"); leaves.createOrReplaceTempView("leaves"); Dataset resultParent = spark .sql( - "SELECT resId as key, " + + "SELECT entityId as key, " + "collect_set(parent) valueSet " + "FROM (SELECT key as child, parent " + " FROM childParent " + @@ -172,7 +171,7 @@ public class StepActions implements Serializable { "JOIN leaves " + "ON leaves.value = cp.child " + "JOIN (" + - "SELECT key as resId, org " + + "SELECT key as entityId, org " + "FROM resultOrg " + "LATERAL VIEW explode (valueSet) ks as org ) as ro " + "ON leaves.value = ro.org " + @@ -186,19 +185,16 @@ public class StepActions implements Serializable { .getValueSet() .stream() .map( - orgId -> getRelation( + orgId -> getAffiliationRelation( v.getKey(), orgId, - ModelConstants.HAS_AUTHOR_INSTITUTION, - ModelConstants.RESULT_ORGANIZATION, - ModelConstants.AFFILIATION, - PROPAGATION_DATA_INFO_TYPE, - PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID, - PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME)) + semantics)) .collect(Collectors.toList()) .iterator(), Encoders.bean(Relation.class)); } + + } diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_preparation_parameter.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_preparation_parameter.json similarity index 87% rename from dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_preparation_parameter.json rename to dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_preparation_parameter.json index c79bfe05d..b59937331 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_preparation_parameter.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_preparation_parameter.json @@ -40,5 +40,11 @@ "paramLongName": "relationPath", "paramDescription": "the path where to store the selected subset of relations", "paramRequired": false + }, + { + "paramName": "pop", + "paramLongName": "projectOrganizationPath", + "paramDescription": "the number of iterations to be computed", + "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_propagation_parameter.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_propagation_parameter.json similarity index 90% rename from dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_propagation_parameter.json rename to dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_propagation_parameter.json index e09cd62fa..5a8597f38 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_propagation_parameter.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_propagation_parameter.json @@ -52,5 +52,10 @@ "paramLongName": "iterations", "paramDescription": "the number of iterations to be computed", "paramRequired": false - } + },{ + "paramName": "pop", + "paramLongName": "projectOrganizationPath", + "paramDescription": "the number of iterations to be computed", + "paramRequired": true +} ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/oozie_app/config-default.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/oozie_app/config-default.xml similarity index 100% rename from dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/oozie_app/config-default.xml rename to dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/oozie_app/config-default.xml diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/oozie_app/workflow.xml similarity index 96% rename from dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/oozie_app/workflow.xml rename to dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/oozie_app/workflow.xml index 5ce2f5c06..ff6ec8f37 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/oozie_app/workflow.xml @@ -134,7 +134,7 @@ yarn cluster PrepareResultOrganizationAssociation - eu.dnetlib.dhp.resulttoorganizationfromsemrel.PrepareInfo + eu.dnetlib.dhp.entitytoorganizationfromsemrel.PrepareInfo dhp-enrichment-${projectVersion}.jar --executor-cores=${sparkExecutorCores} @@ -150,6 +150,7 @@ --leavesPath${workingDir}/preparedInfo/leavesPath --childParentPath${workingDir}/preparedInfo/childParentPath --resultOrgPath${workingDir}/preparedInfo/resultOrgPath + --projectOrganizationPath${workingDir}/preparedInfo/projectOrganizationPath --relationPath${workingDir}/preparedInfo/relation @@ -161,7 +162,7 @@ yarn cluster resultToOrganizationFromSemRel - eu.dnetlib.dhp.resulttoorganizationfromsemrel.SparkResultToOrganizationFromSemRel + eu.dnetlib.dhp.entitytoorganizationfromsemrel.SparkResultToOrganizationFromSemRel dhp-enrichment-${projectVersion}.jar --executor-cores=${sparkExecutorCores} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PrepareInfoJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java similarity index 94% rename from dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PrepareInfoJobTest.java rename to dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java index 2d2668db3..3d7086739 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/PrepareInfoJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java @@ -1,22 +1,17 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; - -import static eu.dnetlib.dhp.PropagationConstant.readPath; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -28,7 +23,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.KeyValueSet; -import eu.dnetlib.dhp.projecttoresult.SparkResultToProjectThroughSemRelJob; import eu.dnetlib.dhp.schema.oaf.Relation; public class PrepareInfoJobTest { @@ -78,11 +72,12 @@ public class PrepareInfoJobTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-graphPath", getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/childparenttest1") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/childparenttest1") .getPath(), "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -223,11 +218,12 @@ public class PrepareInfoJobTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-graphPath", getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/childparenttest2") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/childparenttest2") .getPath(), "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -343,11 +339,12 @@ public class PrepareInfoJobTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-graphPath", getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/resultorganizationtest") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/resultorganizationtest") .getPath(), "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -355,7 +352,7 @@ public class PrepareInfoJobTest { final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/relation") + .textFile(workingDir.toString() + "/relation/result") .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); Dataset verificationDs = spark.createDataset(tmp.rdd(), Encoders.bean(Relation.class)); @@ -373,11 +370,12 @@ public class PrepareInfoJobTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-graphPath", getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/resultorganizationtest") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/resultorganizationtest") .getPath(), "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -507,11 +505,12 @@ public class PrepareInfoJobTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-graphPath", getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/resultorganizationtest") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/resultorganizationtest") .getPath(), "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -534,11 +533,12 @@ public class PrepareInfoJobTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-graphPath", getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/childparenttest1") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/childparenttest1") .getPath(), "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/SparkJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java similarity index 98% rename from dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/SparkJobTest.java rename to dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java index 7dd575b66..a4d8f83e3 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/SparkJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java @@ -1,7 +1,6 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; -import static eu.dnetlib.dhp.PropagationConstant.isSparkSessionManaged; import static eu.dnetlib.dhp.PropagationConstant.readPath; import java.io.IOException; @@ -12,7 +11,6 @@ import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/StepActionsTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java similarity index 99% rename from dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/StepActionsTest.java rename to dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java index 5c715f3b9..77ed4dcbf 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttoorganizationfromsemrel/StepActionsTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.resulttoorganizationfromsemrel; +package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import java.io.IOException; import java.nio.file.Files; diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/childparenttest1/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/childparenttest1/relation similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/childparenttest1/relation rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/childparenttest1/relation diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/childparenttest2/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/childparenttest2/relation similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/childparenttest2/relation rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/childparenttest2/relation diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/childParentOrg/childparent b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/childparent similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/childParentOrg/childparent rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/childparent diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/currentIteration/leaves b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/leaves similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/currentIteration/leaves rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/leaves diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relation similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/relation rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relation diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/relsforiteration1/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relsforiteration1/relation similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/relsforiteration1/relation rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relsforiteration1/relation diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/resultOrganization/resultorganization b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/resultorganization similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/resultOrganization/resultorganization rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/resultorganization diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/resultorganizationtest/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/resultorganizationtest/relation similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/resultorganizationtest/relation rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/resultorganizationtest/relation From 97d72d41c31cc222f50b47a05d4993692892fb3c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 31 May 2023 18:53:22 +0200 Subject: [PATCH 6/8] finalization of implementation and testing --- .../eu/dnetlib/dhp/PropagationConstant.java | 7 + .../PrepareInfo.java | 8 +- .../SparkResultToOrganizationFromSemRel.java | 50 +- .../StepActions.java | 103 +-- .../PrepareInfoJobTest.java | 166 +++++ .../SparkJobTest.java | 648 ++++++++++++++++-- .../StepActionsTest.java | 26 +- .../execstep/graph/project | 7 + .../execstep/graph/result | 7 + .../projectOrganization/projectorganization | 5 + .../execstep/relation | 14 - .../projectorganizationtest/relation | 7 + 12 files changed, 921 insertions(+), 127 deletions(-) create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/project create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/result create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/projectorganization delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relation create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest/relation diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index 53769c9fb..87528ef58 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -174,6 +174,13 @@ public class PropagationConstant { return newRelations; } + public static Relation getRelation(String source, String target, String rel_class){ + if (ModelConstants.HAS_PARTICIPANT.equals(rel_class)){ + return getParticipantRelation(source, target, rel_class); + }else + return getAffiliationRelation(source, target, rel_class); + } + public static Relation getParticipantRelation( String source, String target, diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java index 7ad9c4cee..971ef436f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java @@ -49,7 +49,7 @@ public class PrepareInfo implements Serializable { // associate projects to all the participant orgs private static final String PROJECT_ORGANIZATION_QUERY = "SELECT source key, collect_set(target) as valueSet " + "FROM relation " + - "WHERE lower(relclass) = '" + ModelConstants.IS_PARTICIPANT.toLowerCase() + + "WHERE lower(relclass) = '" + ModelConstants.HAS_PARTICIPANT.toLowerCase() + "' and datainfo.deletedbyinference = false " + "GROUP BY source"; @@ -103,7 +103,7 @@ public class PrepareInfo implements Serializable { } private static void prepareInfo(SparkSession spark, String inputPath, String childParentOrganizationPath, - String currentIterationPath, String resultOrganizationPath, String resultProjectPath, String relationPath) { + String currentIterationPath, String resultOrganizationPath, String projectOrganizationPath, String relationPath) { Dataset relation = readPath(spark, inputPath + "/relation", Relation.class); relation.createOrReplaceTempView("relation"); @@ -129,7 +129,7 @@ public class PrepareInfo implements Serializable { .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") - .json(resultProjectPath); + .json(projectOrganizationPath); relation .filter( @@ -143,7 +143,7 @@ public class PrepareInfo implements Serializable { relation .filter( (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && - r.getRelClass().equals(ModelConstants.IS_PARTICIPANT)) + r.getRelClass().equals(ModelConstants.HAS_PARTICIPANT)) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java index 19e55a905..dd32552ad 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java @@ -39,7 +39,7 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { .toString( SparkResultToOrganizationFromIstRepoJob.class .getResourceAsStream( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/input_propagation_parameter.json")); + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/input_propagation_parameter.json")); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); @@ -129,7 +129,7 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { notReachedFirstParent); doPropagate( - spark, leavesPath, childParentPath, resultOrganizationPath, graphPath, + spark, leavesPath, childParentPath, resultOrganizationPath, projectOrganizationPath, graphPath, workingPath, outputPath, propagationCounter); } @@ -142,20 +142,20 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { StepActions .execStep( spark, graphPath + "/result", workingPath + NEW_RESULT_RELATION_PATH, - leavesPath, childParentPath, resultOrganizationPath); + leavesPath, childParentPath, resultOrganizationPath, ModelConstants.HAS_AUTHOR_INSTITUTION); addNewRelations(spark, workingPath + NEW_RESULT_RELATION_PATH, outputPath); StepActions .execStep( spark, graphPath + "/project", workingPath + NEW_PROJECT_RELATION_PATH, - leavesPath, childParentPath, projectOrganizationPath); + leavesPath, childParentPath, projectOrganizationPath, ModelConstants.HAS_PARTICIPANT); addNewRelations(spark, workingPath + NEW_PROJECT_RELATION_PATH, outputPath); } private static void doPropagate(SparkSession spark, String leavesPath, String childParentPath, - String resultOrganizationPath, String graphPath, String workingPath, String outputPath, + String resultOrganizationPath, String projectOrganizationPath, String graphPath, String workingPath, String outputPath, PropagationCounter propagationCounter) { int iteration = 0; long leavesCount; @@ -164,13 +164,18 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { iteration++; StepActions .execStep( - spark, graphPath, workingPath + NEW_RESULT_RELATION_PATH, - leavesPath, childParentPath, resultOrganizationPath); + spark, graphPath + "/result", workingPath + NEW_RESULT_RELATION_PATH, + leavesPath, childParentPath, resultOrganizationPath, ModelConstants.HAS_AUTHOR_INSTITUTION); + StepActions + .execStep( + spark, graphPath + "/project", workingPath + NEW_PROJECT_RELATION_PATH, + leavesPath, childParentPath, projectOrganizationPath, ModelConstants.HAS_PARTICIPANT); + StepActions .prepareForNextStep( - spark, workingPath + NEW_RESULT_RELATION_PATH, resultOrganizationPath, leavesPath, - childParentPath, workingPath + "/leaves", workingPath + "/resOrg"); - moveOutput(spark, workingPath, leavesPath, resultOrganizationPath); + spark, workingPath , resultOrganizationPath, projectOrganizationPath, leavesPath, + childParentPath, workingPath + "/leaves", workingPath + "/resOrg", workingPath + "/projOrg"); + moveOutput(spark, workingPath, leavesPath, resultOrganizationPath, projectOrganizationPath); leavesCount = readPath(spark, leavesPath, Leaves.class).count(); } while (leavesCount > 0 && iteration < MAX_ITERATION); @@ -199,6 +204,7 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { } addNewRelations(spark, workingPath + NEW_RESULT_RELATION_PATH, outputPath); + addNewRelations(spark, workingPath + NEW_PROJECT_RELATION_PATH, outputPath); } private static void moveOutput(SparkSession spark, String workingPath, String leavesPath, @@ -217,6 +223,28 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { } + private static void moveOutput(SparkSession spark, String workingPath, String leavesPath, + String resultOrganizationPath, String projectOrganizationPath) { + readPath(spark, workingPath + "/leaves", Leaves.class) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(leavesPath); + + readPath(spark, workingPath + "/resOrg", KeyValueSet.class) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(resultOrganizationPath); + + readPath(spark, workingPath + "/projOrg", KeyValueSet.class) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(projectOrganizationPath); + + } + private static void addNewRelations(SparkSession spark, String newRelationPath, String outputPath) { Dataset relation = readPath(spark, newRelationPath, Relation.class); @@ -237,7 +265,7 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { return Arrays .asList( r, getParticipantRelation( - r.getTarget(), r.getSource(), ModelConstants.HAS_PARTICIPANT)) + r.getTarget(), r.getSource(), ModelConstants.IS_PARTICIPANT)) .iterator(); } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java index 5b6c397cf..de5034d38 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java @@ -3,6 +3,8 @@ package eu.dnetlib.dhp.entitytoorganizationfromsemrel; import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.PropagationConstant.readPath; +import static eu.dnetlib.dhp.entitytoorganizationfromsemrel.SparkResultToOrganizationFromSemRel.NEW_PROJECT_RELATION_PATH; +import static eu.dnetlib.dhp.entitytoorganizationfromsemrel.SparkResultToOrganizationFromSemRel.NEW_RESULT_RELATION_PATH; import java.io.Serializable; import java.util.*; @@ -26,13 +28,14 @@ public class StepActions implements Serializable { public static void execStep(SparkSession spark, String graphPath, String newRelationPath, - String leavesPath, String chldParentOrgPath, String entityOrgPath) { + String leavesPath, String chldParentOrgPath, String entityOrgPath, String rel_class) { Dataset relationGraph = readPath(spark, graphPath, Relation.class); // select only the relation source target among those proposed by propagation that are not already existent + getNewRels( newRelationPath, relationGraph, - getPropagationRelation(spark, leavesPath, chldParentOrgPath, entityOrgPath, ModelConstants.HAS_AUTHOR_INSTITUTION)); + getPropagationRelation(spark, leavesPath, chldParentOrgPath, entityOrgPath, rel_class)); } @@ -43,16 +46,30 @@ public class StepActions implements Serializable { changeLeavesSet(spark, leavesPath, chldParentOrgPath, leavesOutputPath); // add the new relations obtained from propagation to the keyvalueset result organization - updateResultOrganization( + updateEntityOrganization( spark, resultOrgPath, readPath(spark, selectedRelsPath, Relation.class), orgOutputPath); } - private static void updateResultOrganization(SparkSession spark, String resultOrgPath, - Dataset selectedRels, String outputPath) { - Dataset resultOrg = readPath(spark, resultOrgPath, KeyValueSet.class); - resultOrg + public static void prepareForNextStep(SparkSession spark, String selectedRelsPath, String resultOrgPath, String projectOrgPath, + String leavesPath, String chldParentOrgPath, String leavesOutputPath, + String orgOutputPath, String outputProjectPath) { + // use of the parents as new leaves set + changeLeavesSet(spark, leavesPath, chldParentOrgPath, leavesOutputPath); + + // add the new relations obtained from propagation to the keyvalueset result organization + updateEntityOrganization( + spark, resultOrgPath, readPath(spark, selectedRelsPath + NEW_RESULT_RELATION_PATH, Relation.class), orgOutputPath); + + updateEntityOrganization( + spark, projectOrgPath, readPath(spark, selectedRelsPath + NEW_PROJECT_RELATION_PATH, Relation.class), outputProjectPath); + } + + private static void updateEntityOrganization(SparkSession spark, String entityOrgPath, + Dataset selectedRels, String outputPath) { + Dataset entityOrg = readPath(spark, entityOrgPath, KeyValueSet.class); + entityOrg .joinWith( - selectedRels, resultOrg + selectedRels, entityOrg .col("key") .equalTo(selectedRels.col("source")), "left") @@ -111,38 +128,45 @@ public class StepActions implements Serializable { // construction of the set) // if at least one relation in the set was not produced by propagation no new relation will be returned + relationDataset - .union(newRels) - .groupByKey((MapFunction) r -> r.getSource() + r.getTarget(), Encoders.STRING()) - .mapGroups((MapGroupsFunction) (k, it) -> { + .union(newRels) + .groupByKey((MapFunction) r -> r.getSource() + r.getTarget(), Encoders.STRING()) + .mapGroups((MapGroupsFunction) (k, it) -> { - ArrayList relationList = new ArrayList<>(); - relationList.add(it.next()); - it.forEachRemaining(rel -> relationList.add(rel)); + ArrayList relationList = new ArrayList<>(); + relationList.add(it.next()); + it.forEachRemaining(rel -> relationList.add(rel)); - if (relationList - .stream() - .filter( - rel -> !rel - .getDataInfo() - .getProvenanceaction() - .getClassid() - .equals(PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID)) - .count() > 0) { - return null; - } + if (relationList + .stream() + .filter( + rel -> !rel + .getDataInfo() + .getProvenanceaction() + .getClassid() + .equals(PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID) && !rel + .getDataInfo() + .getProvenanceaction() + .getClassid() + .equals(PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID)) + .count() > 0) { + return null; + } + + return new ObjectMapper().writeValueAsString(relationList.get(0)); + + }, Encoders.STRING()) + .filter(Objects::nonNull) + .map( + (MapFunction) r -> new ObjectMapper().readValue(r, Relation.class), + Encoders.bean(Relation.class)) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(newRelationPath); - return new ObjectMapper().writeValueAsString(relationList.get(0)); - }, Encoders.STRING()) - .filter(Objects::nonNull) - .map( - (MapFunction) r -> new ObjectMapper().readValue(r, Relation.class), - Encoders.bean(Relation.class)) - .write() - .mode(SaveMode.Append) - .option("compression", "gzip") - .json(newRelationPath); } @@ -172,20 +196,21 @@ public class StepActions implements Serializable { "ON leaves.value = cp.child " + "JOIN (" + "SELECT key as entityId, org " + - "FROM resultOrg " + + "FROM entityOrg " + "LATERAL VIEW explode (valueSet) ks as org ) as ro " + "ON leaves.value = ro.org " + - "GROUP BY resId") + "GROUP BY entityId") .as(Encoders.bean(KeyValueSet.class)); - // create new relations from result to organization for each result linked to a leaf + + // create new relations from entity to organization for each entity linked to a leaf return resultParent .flatMap( (FlatMapFunction) v -> v .getValueSet() .stream() .map( - orgId -> getAffiliationRelation( + orgId -> getRelation( v.getKey(), orgId, semantics)) diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java index 3d7086739..f29e8d24a 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java @@ -361,6 +361,37 @@ public class PrepareInfoJobTest { } + @Test + public void relationProjectTest() throws Exception { + + PrepareInfo + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-graphPath", getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest") + .getPath(), + "-hive_metastore_uris", "", + "-leavesPath", workingDir.toString() + "/currentIteration/", + "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-childParentPath", workingDir.toString() + "/childParentOrg/", + "-relationPath", workingDir.toString() + "/relation" + + }); + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/relation/project") + .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); + + Dataset verificationDs = spark.createDataset(tmp.rdd(), Encoders.bean(Relation.class)); + + Assertions.assertEquals(7, verificationDs.count()); + + } + @Test public void resultOrganizationTest1() throws Exception { @@ -496,6 +527,141 @@ public class PrepareInfoJobTest { } + @Test + public void projectOrganizationTest1() throws Exception { + + PrepareInfo + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-graphPath", getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest") + .getPath(), + "-hive_metastore_uris", "", + "-leavesPath", workingDir.toString() + "/currentIteration/", + "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-childParentPath", workingDir.toString() + "/childParentOrg/", + "-relationPath", workingDir.toString() + "/relation" + + }); + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/projectOrganization/") + .map(item -> OBJECT_MAPPER.readValue(item, KeyValueSet.class)); + + Dataset verificationDs = spark.createDataset(tmp.rdd(), Encoders.bean(KeyValueSet.class)); + + Assertions.assertEquals(5, verificationDs.count()); + + Assertions + .assertEquals( + 2, verificationDs + .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") + .collectAsList() + .get(0) + .getValueSet() + .size()); + Assertions + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + Assertions + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|pippo_wf_001::2899e571609779168222fdeb59cb916d")); + + Assertions + .assertEquals( + 2, verificationDs + .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") + .collectAsList() + .get(0) + .getValueSet() + .size()); + Assertions + .assertTrue( + verificationDs + .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0")); + Assertions + .assertTrue( + verificationDs + .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|pippo_wf_001::2899e571609779168222fdeb59cb916d")); + + Assertions + .assertEquals( + 1, verificationDs + .filter("key = '40|doajarticles::03748bcb5d754c951efec9700e18a56d'") + .collectAsList() + .get(0) + .getValueSet() + .size()); + Assertions + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::03748bcb5d754c951efec9700e18a56d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + + Assertions + .assertEquals( + 1, verificationDs + .filter("key = '40|openaire____::ec653e804967133b9436fdd30d3ff51d'") + .collectAsList() + .get(0) + .getValueSet() + .size()); + Assertions + .assertTrue( + verificationDs + .filter("key = '40|openaire____::ec653e804967133b9436fdd30d3ff51d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + + Assertions + .assertEquals( + 1, verificationDs + .filter("key = '40|doajarticles::1cae0b82b56ccd97c2db1f698def7074'") + .collectAsList() + .get(0) + .getValueSet() + .size()); + Assertions + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::1cae0b82b56ccd97c2db1f698def7074'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1")); + + verificationDs + .foreach((ForeachFunction) v -> System.out.println(OBJECT_MAPPER.writeValueAsString(v))); + + } + @Test public void foundLeavesTest1() throws Exception { diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java index a4d8f83e3..eb4ade0da 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java @@ -65,25 +65,30 @@ public class SparkJobTest { } @Test - public void completeExecution() throws Exception { + public void completeResultExecution() throws Exception { final String graphPath = getClass() - .getResource("/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep") + .getResource("/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph") .getPath(); final String leavesPath = getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/currentIteration/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") .getPath(); final String childParentPath = getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/childParentOrg/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") .getPath(); final String resultOrgPath = getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/resultOrganization/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") .getPath(); + final String projectOrgPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") + .getPath(); + readPath(spark, leavesPath, Leaves.class) .write() .option("compression", "gzip") @@ -94,6 +99,11 @@ public class SparkJobTest { .option("compression", "gzip") .json(workingDir.toString() + "/orgsInput"); + readPath(spark, projectOrgPath, KeyValueSet.class) + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/projectInput"); + SparkResultToOrganizationFromSemRel .main( @@ -104,95 +114,97 @@ public class SparkJobTest { "-outputPath", workingDir.toString() + "/finalrelation", "-leavesPath", workingDir.toString() + "/leavesInput", "-resultOrgPath", workingDir.toString() + "/orgsInput", + "-projectOrganizationPath", workingDir.toString() + "/projectInput", "-childParentPath", childParentPath, "-workingDir", workingDir.toString() }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - JavaRDD tmp = sc + JavaRDD temp = sc .textFile(workingDir.toString() + "/finalrelation") .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); - tmp.foreach(r -> System.out.println(OBJECT_MAPPER.writeValueAsString(r))); + Assertions.assertEquals(36, temp.count()); - Assertions.assertEquals(18, tmp.count()); - tmp.foreach(r -> Assertions.assertEquals(ModelConstants.AFFILIATION, r.getSubRelType())); - tmp.foreach(r -> Assertions.assertEquals(ModelConstants.RESULT_ORGANIZATION, r.getRelType())); - tmp + JavaRDD result = temp.filter(r -> r.getSource().startsWith("50|") || r.getTarget().startsWith("50|")); + Assertions.assertEquals(18, result.count()); + result.foreach(r -> Assertions.assertEquals(ModelConstants.AFFILIATION, r.getSubRelType())); + result.foreach(r -> Assertions.assertEquals(ModelConstants.RESULT_ORGANIZATION, r.getRelType())); + result .foreach( r -> Assertions .assertEquals( PropagationConstant.PROPAGATION_DATA_INFO_TYPE, r.getDataInfo().getInferenceprovenance())); - tmp + result .foreach( r -> Assertions .assertEquals( PropagationConstant.PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID, r.getDataInfo().getProvenanceaction().getClassid())); - tmp + result .foreach( r -> Assertions .assertEquals( PropagationConstant.PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME, r.getDataInfo().getProvenanceaction().getClassname())); - tmp + result .foreach( r -> Assertions .assertEquals( "0.85", r.getDataInfo().getTrust())); - Assertions.assertEquals(9, tmp.filter(r -> r.getSource().substring(0, 3).equals("50|")).count()); - tmp + Assertions.assertEquals(9, result.filter(r -> r.getSource().substring(0, 3).equals("50|")).count()); + result .filter(r -> r.getSource().substring(0, 3).equals("50|")) .foreach(r -> Assertions.assertEquals(ModelConstants.HAS_AUTHOR_INSTITUTION, r.getRelClass())); Assertions .assertEquals( - 2, tmp.filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + 2, result.filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); Assertions .assertEquals( - 3, tmp.filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + 3, result.filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); Assertions .assertEquals( - 2, tmp.filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + 2, result.filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); Assertions .assertEquals( - 1, tmp.filter(r -> r.getSource().equals("50|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + 1, result.filter(r -> r.getSource().equals("50|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); Assertions .assertEquals( - 1, tmp.filter(r -> r.getSource().equals("50|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + 1, result.filter(r -> r.getSource().equals("50|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); - Assertions.assertEquals(9, tmp.filter(r -> r.getSource().substring(0, 3).equals("20|")).count()); - tmp + Assertions.assertEquals(9, result.filter(r -> r.getSource().substring(0, 3).equals("20|")).count()); + result .filter(r -> r.getSource().substring(0, 3).equals("20|")) .foreach(r -> Assertions.assertEquals(ModelConstants.IS_AUTHOR_INSTITUTION_OF, r.getRelClass())); Assertions .assertEquals( - 1, tmp.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + 1, result.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); Assertions .assertEquals( - 1, tmp.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + 1, result.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); Assertions .assertEquals( - 2, tmp.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + 2, result.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); Assertions .assertEquals( - 2, tmp.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + 2, result.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); Assertions .assertEquals( - 3, tmp.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + 3, result.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) .map(r -> r.getTarget()) .collect() .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) .map(r -> r.getTarget()) .collect() @@ -200,14 +212,14 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) .map(r -> r.getTarget()) .collect() .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) .map(r -> r.getTarget()) .collect() @@ -215,21 +227,21 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")) .map(r -> r.getTarget()) .collect() .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")) .map(r -> r.getTarget()) .collect() .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")) .map(r -> r.getTarget()) .collect() @@ -237,7 +249,7 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|openaire____::ec653e804967133b9436fdd30d3ff51d")) .map(r -> r.getTarget()) .collect() @@ -245,7 +257,7 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("50|doajarticles::03748bcb5d754c951efec9700e18a56d")) .map(r -> r.getTarget()) .collect() @@ -253,14 +265,14 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) .map(r -> r.getTarget()) .collect() .contains("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) .map(r -> r.getTarget()) .collect() @@ -268,14 +280,14 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) .map(r -> r.getTarget()) .collect() .contains("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) .map(r -> r.getTarget()) .collect() @@ -283,21 +295,21 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) .map(r -> r.getTarget()) .collect() .contains("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) .map(r -> r.getTarget()) .collect() .contains("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) .map(r -> r.getTarget()) .collect() @@ -305,7 +317,7 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")) .map(r -> r.getTarget()) .collect() @@ -313,11 +325,555 @@ public class SparkJobTest { Assertions .assertTrue( - tmp + result .filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) .map(r -> r.getTarget()) .collect() .contains("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); } + @Test + public void completeProjectExecution() throws Exception { + + final String graphPath = getClass() + .getResource("/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph") + .getPath(); + final String leavesPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") + .getPath(); + final String childParentPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") + .getPath(); + + final String resultOrgPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") + .getPath(); + + final String projectOrgPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") + .getPath(); + + readPath(spark, leavesPath, Leaves.class) + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/leavesInput"); + + readPath(spark, resultOrgPath, KeyValueSet.class) + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/orgsInput"); + + readPath(spark, projectOrgPath, KeyValueSet.class) + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/projectInput"); + + SparkResultToOrganizationFromSemRel + + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-relationPath", graphPath, + "-hive_metastore_uris", "", + "-outputPath", workingDir.toString() + "/finalrelation", + "-leavesPath", workingDir.toString() + "/leavesInput", + "-resultOrgPath", workingDir.toString() + "/orgsInput", + "-projectOrganizationPath", workingDir.toString() + "/projectInput", + "-childParentPath", childParentPath, + "-workingDir", workingDir.toString() + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD temp = sc + .textFile(workingDir.toString() + "/finalrelation") + .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); + + Assertions.assertEquals(36, temp.count()); + + JavaRDD project = temp.filter(r -> r.getSource().startsWith("40|") || r.getTarget().startsWith("40|")); + Assertions.assertEquals(18, project.count()); + + project.foreach(r -> Assertions.assertEquals(ModelConstants.PARTICIPATION, r.getSubRelType())); + project.foreach(r -> Assertions.assertEquals(ModelConstants.PROJECT_ORGANIZATION, r.getRelType())); + project + .foreach( + r -> Assertions + .assertEquals( + PropagationConstant.PROPAGATION_DATA_INFO_TYPE, r.getDataInfo().getInferenceprovenance())); + project + .foreach( + r -> Assertions + .assertEquals( + PropagationConstant.PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID, + r.getDataInfo().getProvenanceaction().getClassid())); + project + .foreach( + r -> Assertions + .assertEquals( + PropagationConstant.PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_NAME, + r.getDataInfo().getProvenanceaction().getClassname())); + project + .foreach( + r -> Assertions + .assertEquals( + "0.85", + r.getDataInfo().getTrust())); + + Assertions.assertEquals(9, project.filter(r -> r.getSource().substring(0, 3).equals("40|")).count()); + project + .filter(r -> r.getSource().substring(0, 3).equals("40|")) + .foreach(r -> Assertions.assertEquals(ModelConstants.HAS_PARTICIPANT, r.getRelClass())); + Assertions + .assertEquals( + 2, project.filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + Assertions + .assertEquals( + 3, project.filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + Assertions + .assertEquals( + 2, project.filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + Assertions + .assertEquals( + 1, project.filter(r -> r.getSource().equals("40|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + Assertions + .assertEquals( + 1, project.filter(r -> r.getSource().equals("40|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + + Assertions.assertEquals(9, project.filter(r -> r.getSource().substring(0, 3).equals("20|")).count()); + project + .filter(r -> r.getSource().substring(0, 3).equals("20|")) + .foreach(r -> Assertions.assertEquals(ModelConstants.IS_PARTICIPANT, r.getRelClass())); + Assertions + .assertEquals( + 1, project.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + Assertions + .assertEquals( + 1, project.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + Assertions + .assertEquals( + 2, project.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + Assertions + .assertEquals( + 2, project.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + Assertions + .assertEquals( + 3, project.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) + .map(r -> r.getTarget()) + .collect() + .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|openaire____::ec653e804967133b9436fdd30d3ff51d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|openaire____::ec653e804967133b9436fdd30d3ff51d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::03748bcb5d754c951efec9700e18a56d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + + Assertions + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + } + + @Test + public void singleIterationExecution() throws Exception { + + final String graphPath = getClass() + .getResource("/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph") + .getPath(); + final String leavesPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") + .getPath(); + final String childParentPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") + .getPath(); + + final String resultOrgPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") + .getPath(); + + final String projectOrgPath = getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") + .getPath(); + + readPath(spark, leavesPath, Leaves.class) + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/leavesInput"); + + readPath(spark, resultOrgPath, KeyValueSet.class) + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/orgsInput"); + + readPath(spark, projectOrgPath, KeyValueSet.class) + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/projectInput"); + + SparkResultToOrganizationFromSemRel + + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-relationPath", graphPath, + "-hive_metastore_uris", "", + "-outputPath", workingDir.toString() + "/finalrelation", + "-leavesPath", workingDir.toString() + "/leavesInput", + "-resultOrgPath", workingDir.toString() + "/orgsInput", + "-projectOrganizationPath", workingDir.toString() + "/projectInput", + "-childParentPath", childParentPath, + "-workingDir", workingDir.toString(), + "-iterations", "1" + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD temp = sc + .textFile(workingDir.toString() + "/finalrelation") + .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); + + Assertions.assertEquals(16, temp.count()); + + Assertions.assertEquals(4, temp.filter(r -> r.getSource().startsWith("50|")).count()); + Assertions.assertEquals(4, temp.filter(r -> r.getTarget().startsWith("50|")).count()); + Assertions.assertEquals(4, temp.filter(r -> r.getSource().startsWith("40|")).count()); + Assertions.assertEquals(4, temp.filter(r -> r.getTarget().startsWith("40|")).count()); + Assertions.assertEquals(8, temp.filter(r -> r.getSource().startsWith("20|")).count()); + Assertions.assertEquals(8, temp.filter(r -> r.getSource().startsWith("20|")).count()); + +// JavaRDD result = temp.filter(r -> r.getSource().startsWith("50|") || r.getTarget().startsWith("50|")); +// Assertions.assertEquals(18, result.count()); +// result.foreach(r -> Assertions.assertEquals(ModelConstants.AFFILIATION, r.getSubRelType())); +// result.foreach(r -> Assertions.assertEquals(ModelConstants.RESULT_ORGANIZATION, r.getRelType())); +// result +// .foreach( +// r -> Assertions +// .assertEquals( +// PropagationConstant.PROPAGATION_DATA_INFO_TYPE, r.getDataInfo().getInferenceprovenance())); +// result +// .foreach( +// r -> Assertions +// .assertEquals( +// PropagationConstant.PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID, +// r.getDataInfo().getProvenanceaction().getClassid())); +// result +// .foreach( +// r -> Assertions +// .assertEquals( +// PropagationConstant.PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME, +// r.getDataInfo().getProvenanceaction().getClassname())); +// result +// .foreach( +// r -> Assertions +// .assertEquals( +// "0.85", +// r.getDataInfo().getTrust())); +// +// Assertions.assertEquals(9, result.filter(r -> r.getSource().substring(0, 3).equals("50|")).count()); +// result +// .filter(r -> r.getSource().substring(0, 3).equals("50|")) +// .foreach(r -> Assertions.assertEquals(ModelConstants.HAS_AUTHOR_INSTITUTION, r.getRelClass())); +// Assertions +// .assertEquals( +// 2, result.filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); +// Assertions +// .assertEquals( +// 3, result.filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); +// Assertions +// .assertEquals( +// 2, result.filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); +// Assertions +// .assertEquals( +// 1, result.filter(r -> r.getSource().equals("50|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); +// Assertions +// .assertEquals( +// 1, result.filter(r -> r.getSource().equals("50|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); +// +// Assertions.assertEquals(9, result.filter(r -> r.getSource().substring(0, 3).equals("20|")).count()); +// result +// .filter(r -> r.getSource().substring(0, 3).equals("20|")) +// .foreach(r -> Assertions.assertEquals(ModelConstants.IS_AUTHOR_INSTITUTION_OF, r.getRelClass())); +// Assertions +// .assertEquals( +// 1, result.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); +// Assertions +// .assertEquals( +// 1, result.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); +// Assertions +// .assertEquals( +// 2, result.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); +// Assertions +// .assertEquals( +// 2, result.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); +// Assertions +// .assertEquals( +// 3, result.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|openaire____::ec653e804967133b9436fdd30d3ff51d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("50|doajarticles::03748bcb5d754c951efec9700e18a56d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|openaire____::ec653e804967133b9436fdd30d3ff51d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|doajarticles::03748bcb5d754c951efec9700e18a56d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")); +// +// Assertions +// .assertTrue( +// result +// .filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) +// .map(r -> r.getTarget()) +// .collect() +// .contains("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + } } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java index 77ed4dcbf..7a71240b2 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java @@ -73,21 +73,21 @@ public class StepActionsTest { .execStep( spark, getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/result") .getPath(), workingDir.toString() + "/newRelationPath", getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/currentIteration/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/childParentOrg/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/resultOrganization/") - .getPath()); + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") + .getPath(), ModelConstants.HAS_AUTHOR_INSTITUTION); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); @@ -203,19 +203,19 @@ public class StepActionsTest { spark, getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/relsforiteration1/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relsforiteration1/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/resultOrganization/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/currentIteration/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/childParentOrg/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") .getPath(), workingDir.toString() + "/tempLeaves", workingDir.toString() + "/tempOrgs"); @@ -248,19 +248,19 @@ public class StepActionsTest { spark, getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/relsforiteration1/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relsforiteration1/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/resultOrganization/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/currentIteration/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") .getPath(), getClass() .getResource( - "/eu/dnetlib/dhp/resulttoorganizationfromsemrel/execstep/childParentOrg/") + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") .getPath(), workingDir.toString() + "/tempLeaves", workingDir.toString() + "/tempOrgs"); diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/project b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/project new file mode 100644 index 000000000..e8e35f555 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/project @@ -0,0 +1,7 @@ +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|dedup_wf_001::2899e571609779168222fdeb59cb916d","validated":false} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d","validated":false} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d","validated":false} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0","validated":false} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::03748bcb5d754c951efec9700e18a56d","subRelType":"provision","target":"20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","validated":false} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|openaire____::ec653e804967133b9436fdd30d3ff51d","subRelType":"provision","target":"20|doajarticles::1cae0b82b56ccd97c2db1f698def7074","validated":false} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::1cae0b82b56ccd97c2db1f698def7074","subRelType":"provision","target":"20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1","validated":false} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/result b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/result new file mode 100644 index 000000000..5aeabb71b --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph/result @@ -0,0 +1,7 @@ +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|dedup_wf_001::2899e571609779168222fdeb59cb916d"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::03748bcb5d754c951efec9700e18a56d","subRelType":"provision","target":"20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|openaire____::ec653e804967133b9436fdd30d3ff51d","subRelType":"provision","target":"20|doajarticles::1cae0b82b56ccd97c2db1f698def7074"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::1cae0b82b56ccd97c2db1f698def7074","subRelType":"provision","target":"20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1"} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/projectorganization b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/projectorganization new file mode 100644 index 000000000..81803f29d --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/projectorganization @@ -0,0 +1,5 @@ +{"key":"40|openaire____::ec653e804967133b9436fdd30d3ff51d","valueSet":["20|doajarticles::1cae0b82b56ccd97c2db1f698def7074"]} +{"key":"40|doajarticles::03748bcb5d754c951efec9700e18a56d","valueSet":["20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f"]} +{"key":"40|doajarticles::1cae0b82b56ccd97c2db1f698def7074","valueSet":["20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1"]} +{"key":"40|dedup_wf_001::2899e571609779168222fdeb59cb916d","valueSet":["20|pippo_wf_001::2899e571609779168222fdeb59cb916d","20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0"]} +{"key":"40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","valueSet":["20|pippo_wf_001::2899e571609779168222fdeb59cb916d","20|dedup_wf_001::2899e571609779168222fdeb59cb916d"]} diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relation deleted file mode 100644 index db7db8fdd..000000000 --- a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/relation +++ /dev/null @@ -1,14 +0,0 @@ -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"isparentof","relType":"datasourceOrganization","source":"20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|dedup_wf_001::2899e571609779168222fdeb59cb916d"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"isparentof","relType":"datasourceOrganization","source":"20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"isparentof","relType":"datasourceOrganization","source":"20|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"isparentof","relType":"datasourceOrganization","source":"20|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"isparentof","relType":"datasourceOrganization","source":"20|doajarticles::03748bcb5d754c951efec9700e18a56d","subRelType":"provision","target":"20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"isparentof","relType":"datasourceOrganization","source":"20|openaire____::ec653e804967133b9436fdd30d3ff51d","subRelType":"provision","target":"20|doajarticles::1cae0b82b56ccd97c2db1f698def7074"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"isparentof","relType":"datasourceOrganization","source":"20|doajarticles::1cae0b82b56ccd97c2db1f698def7074","subRelType":"provision","target":"20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|dedup_wf_001::2899e571609779168222fdeb59cb916d"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::03748bcb5d754c951efec9700e18a56d","subRelType":"provision","target":"20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|openaire____::ec653e804967133b9436fdd30d3ff51d","subRelType":"provision","target":"20|doajarticles::1cae0b82b56ccd97c2db1f698def7074"} -{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasAuthorInstitution","relType":"datasourceOrganization","source":"50|doajarticles::1cae0b82b56ccd97c2db1f698def7074","subRelType":"provision","target":"20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1"} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest/relation new file mode 100644 index 000000000..10d46b1cb --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest/relation @@ -0,0 +1,7 @@ +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|dedup_wf_001::2899e571609779168222fdeb59cb916d"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|pippo_wf_001::2899e571609779168222fdeb59cb916d"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|dedup_wf_001::2899e571609779168222fdeb59cb916d","subRelType":"provision","target":"20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::03748bcb5d754c951efec9700e18a56d","subRelType":"provision","target":"20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|openaire____::ec653e804967133b9436fdd30d3ff51d","subRelType":"provision","target":"20|doajarticles::1cae0b82b56ccd97c2db1f698def7074"} +{"collectedfrom":[{"key":"10|driver______::bee53aa31dc2cbb538c10c2b65fa5824","value":"DOAJ-Articles"}],"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":1592688952862,"properties":[],"relClass":"hasParticipant","relType":"datasourceOrganization","source":"40|doajarticles::1cae0b82b56ccd97c2db1f698def7074","subRelType":"provision","target":"20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1"} \ No newline at end of file From daf4d7971b6b1a355b8930b4a2e1f4fc25efd380 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 31 May 2023 18:56:58 +0200 Subject: [PATCH 7/8] refactoring --- .../eu/dnetlib/dhp/PropagationConstant.java | 48 +- .../PrepareInfo.java | 50 +- .../SparkResultToOrganizationFromSemRel.java | 70 ++- .../StepActions.java | 88 ++- .../PrepareInfoJobTest.java | 246 ++++----- .../SparkJobTest.java | 509 +++++++++--------- .../StepActionsTest.java | 3 +- 7 files changed, 518 insertions(+), 496 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index 87528ef58..053300696 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -174,37 +174,39 @@ public class PropagationConstant { return newRelations; } - public static Relation getRelation(String source, String target, String rel_class){ - if (ModelConstants.HAS_PARTICIPANT.equals(rel_class)){ + public static Relation getRelation(String source, String target, String rel_class) { + if (ModelConstants.HAS_PARTICIPANT.equals(rel_class)) { return getParticipantRelation(source, target, rel_class); - }else + } else return getAffiliationRelation(source, target, rel_class); } public static Relation getParticipantRelation( - String source, - String target, - String rel_class) { - return getRelation(source, target , - rel_class, - ModelConstants.PROJECT_ORGANIZATION, - ModelConstants.PARTICIPATION, - PROPAGATION_DATA_INFO_TYPE, - PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID, - PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_NAME); + String source, + String target, + String rel_class) { + return getRelation( + source, target, + rel_class, + ModelConstants.PROJECT_ORGANIZATION, + ModelConstants.PARTICIPATION, + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID, + PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_NAME); } public static Relation getAffiliationRelation( - String source, - String target, - String rel_class) { - return getRelation(source, target , - rel_class, - ModelConstants.RESULT_ORGANIZATION, - ModelConstants.AFFILIATION, - PROPAGATION_DATA_INFO_TYPE, - PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID, - PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME); + String source, + String target, + String rel_class) { + return getRelation( + source, target, + rel_class, + ModelConstants.RESULT_ORGANIZATION, + ModelConstants.AFFILIATION, + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID, + PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_NAME); } public static Relation getRelation( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java index 971ef436f..8d3432f06 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfo.java @@ -7,7 +7,6 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; import java.io.Serializable; import java.util.*; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.*; @@ -15,6 +14,8 @@ import org.apache.spark.sql.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + import eu.dnetlib.dhp.KeyValueSet; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob; @@ -48,10 +49,10 @@ public class PrepareInfo implements Serializable { // associate projects to all the participant orgs private static final String PROJECT_ORGANIZATION_QUERY = "SELECT source key, collect_set(target) as valueSet " + - "FROM relation " + - "WHERE lower(relclass) = '" + ModelConstants.HAS_PARTICIPANT.toLowerCase() + - "' and datainfo.deletedbyinference = false " + - "GROUP BY source"; + "FROM relation " + + "WHERE lower(relclass) = '" + ModelConstants.HAS_PARTICIPANT.toLowerCase() + + "' and datainfo.deletedbyinference = false " + + "GROUP BY source"; public static void main(String[] args) throws Exception { @@ -98,12 +99,13 @@ public class PrepareInfo implements Serializable { childParentPath, leavesPath, resultOrganizationPath, - projectOrgPath, + projectOrgPath, relationPath)); } private static void prepareInfo(SparkSession spark, String inputPath, String childParentOrganizationPath, - String currentIterationPath, String resultOrganizationPath, String projectOrganizationPath, String relationPath) { + String currentIterationPath, String resultOrganizationPath, String projectOrganizationPath, + String relationPath) { Dataset relation = readPath(spark, inputPath + "/relation", Relation.class); relation.createOrReplaceTempView("relation"); @@ -124,30 +126,30 @@ public class PrepareInfo implements Serializable { .json(resultOrganizationPath); spark - .sql(PROJECT_ORGANIZATION_QUERY) - .as(Encoders.bean(KeyValueSet.class)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(projectOrganizationPath); + .sql(PROJECT_ORGANIZATION_QUERY) + .as(Encoders.bean(KeyValueSet.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(projectOrganizationPath); relation - .filter( - (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && - r.getRelClass().equals(ModelConstants.HAS_AUTHOR_INSTITUTION)) - .write() + .filter( + (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && + r.getRelClass().equals(ModelConstants.HAS_AUTHOR_INSTITUTION)) + .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") .json(relationPath + "/result"); relation - .filter( - (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && - r.getRelClass().equals(ModelConstants.HAS_PARTICIPANT)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(relationPath + "/project"); + .filter( + (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && + r.getRelClass().equals(ModelConstants.HAS_PARTICIPANT)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(relationPath + "/project"); Dataset children = spark .sql( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java index dd32552ad..27e502aba 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkResultToOrganizationFromSemRel.java @@ -92,7 +92,7 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { leavesPath, childParentPath, resultOrganizationPath, - projectOrganizationPath, + projectOrganizationPath, relationPath, workingPath, outputPath, @@ -147,15 +147,16 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { addNewRelations(spark, workingPath + NEW_RESULT_RELATION_PATH, outputPath); StepActions - .execStep( - spark, graphPath + "/project", workingPath + NEW_PROJECT_RELATION_PATH, - leavesPath, childParentPath, projectOrganizationPath, ModelConstants.HAS_PARTICIPANT); + .execStep( + spark, graphPath + "/project", workingPath + NEW_PROJECT_RELATION_PATH, + leavesPath, childParentPath, projectOrganizationPath, ModelConstants.HAS_PARTICIPANT); addNewRelations(spark, workingPath + NEW_PROJECT_RELATION_PATH, outputPath); } private static void doPropagate(SparkSession spark, String leavesPath, String childParentPath, - String resultOrganizationPath, String projectOrganizationPath, String graphPath, String workingPath, String outputPath, + String resultOrganizationPath, String projectOrganizationPath, String graphPath, String workingPath, + String outputPath, PropagationCounter propagationCounter) { int iteration = 0; long leavesCount; @@ -167,13 +168,13 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { spark, graphPath + "/result", workingPath + NEW_RESULT_RELATION_PATH, leavesPath, childParentPath, resultOrganizationPath, ModelConstants.HAS_AUTHOR_INSTITUTION); StepActions - .execStep( - spark, graphPath + "/project", workingPath + NEW_PROJECT_RELATION_PATH, - leavesPath, childParentPath, projectOrganizationPath, ModelConstants.HAS_PARTICIPANT); + .execStep( + spark, graphPath + "/project", workingPath + NEW_PROJECT_RELATION_PATH, + leavesPath, childParentPath, projectOrganizationPath, ModelConstants.HAS_PARTICIPANT); StepActions .prepareForNextStep( - spark, workingPath , resultOrganizationPath, projectOrganizationPath, leavesPath, + spark, workingPath, resultOrganizationPath, projectOrganizationPath, leavesPath, childParentPath, workingPath + "/leaves", workingPath + "/resOrg", workingPath + "/projOrg"); moveOutput(spark, workingPath, leavesPath, resultOrganizationPath, projectOrganizationPath); leavesCount = readPath(spark, leavesPath, Leaves.class).count(); @@ -224,24 +225,24 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { } private static void moveOutput(SparkSession spark, String workingPath, String leavesPath, - String resultOrganizationPath, String projectOrganizationPath) { + String resultOrganizationPath, String projectOrganizationPath) { readPath(spark, workingPath + "/leaves", Leaves.class) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(leavesPath); + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(leavesPath); readPath(spark, workingPath + "/resOrg", KeyValueSet.class) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(resultOrganizationPath); + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(resultOrganizationPath); readPath(spark, workingPath + "/projOrg", KeyValueSet.class) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(projectOrganizationPath); + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(projectOrganizationPath); } @@ -253,25 +254,22 @@ public class SparkResultToOrganizationFromSemRel implements Serializable { .mapGroups( (MapGroupsFunction) (k, it) -> it.next(), Encoders.bean(Relation.class)) .flatMap( - (FlatMapFunction) r -> - { - if(r.getSource().startsWith("50|")){ + (FlatMapFunction) r -> { + if (r.getSource().startsWith("50|")) { return Arrays - .asList( - r, getAffiliationRelation( - r.getTarget(), r.getSource(), ModelConstants.IS_AUTHOR_INSTITUTION_OF)) - .iterator(); - }else{ + .asList( + r, getAffiliationRelation( + r.getTarget(), r.getSource(), ModelConstants.IS_AUTHOR_INSTITUTION_OF)) + .iterator(); + } else { return Arrays - .asList( - r, getParticipantRelation( - r.getTarget(), r.getSource(), ModelConstants.IS_PARTICIPANT)) - .iterator(); + .asList( + r, getParticipantRelation( + r.getTarget(), r.getSource(), ModelConstants.IS_PARTICIPANT)) + .iterator(); } } - - , Encoders.bean(Relation.class)) .write() diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java index de5034d38..386ea1a5c 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActions.java @@ -50,22 +50,25 @@ public class StepActions implements Serializable { spark, resultOrgPath, readPath(spark, selectedRelsPath, Relation.class), orgOutputPath); } - public static void prepareForNextStep(SparkSession spark, String selectedRelsPath, String resultOrgPath, String projectOrgPath, - String leavesPath, String chldParentOrgPath, String leavesOutputPath, - String orgOutputPath, String outputProjectPath) { + public static void prepareForNextStep(SparkSession spark, String selectedRelsPath, String resultOrgPath, + String projectOrgPath, + String leavesPath, String chldParentOrgPath, String leavesOutputPath, + String orgOutputPath, String outputProjectPath) { // use of the parents as new leaves set changeLeavesSet(spark, leavesPath, chldParentOrgPath, leavesOutputPath); // add the new relations obtained from propagation to the keyvalueset result organization updateEntityOrganization( - spark, resultOrgPath, readPath(spark, selectedRelsPath + NEW_RESULT_RELATION_PATH, Relation.class), orgOutputPath); + spark, resultOrgPath, readPath(spark, selectedRelsPath + NEW_RESULT_RELATION_PATH, Relation.class), + orgOutputPath); updateEntityOrganization( - spark, projectOrgPath, readPath(spark, selectedRelsPath + NEW_PROJECT_RELATION_PATH, Relation.class), outputProjectPath); + spark, projectOrgPath, readPath(spark, selectedRelsPath + NEW_PROJECT_RELATION_PATH, Relation.class), + outputProjectPath); } private static void updateEntityOrganization(SparkSession spark, String entityOrgPath, - Dataset selectedRels, String outputPath) { + Dataset selectedRels, String outputPath) { Dataset entityOrg = readPath(spark, entityOrgPath, KeyValueSet.class); entityOrg .joinWith( @@ -128,45 +131,43 @@ public class StepActions implements Serializable { // construction of the set) // if at least one relation in the set was not produced by propagation no new relation will be returned - relationDataset - .union(newRels) - .groupByKey((MapFunction) r -> r.getSource() + r.getTarget(), Encoders.STRING()) - .mapGroups((MapGroupsFunction) (k, it) -> { + .union(newRels) + .groupByKey((MapFunction) r -> r.getSource() + r.getTarget(), Encoders.STRING()) + .mapGroups((MapGroupsFunction) (k, it) -> { - ArrayList relationList = new ArrayList<>(); - relationList.add(it.next()); - it.forEachRemaining(rel -> relationList.add(rel)); + ArrayList relationList = new ArrayList<>(); + relationList.add(it.next()); + it.forEachRemaining(rel -> relationList.add(rel)); - if (relationList - .stream() - .filter( - rel -> !rel - .getDataInfo() - .getProvenanceaction() - .getClassid() - .equals(PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID) && !rel - .getDataInfo() - .getProvenanceaction() - .getClassid() - .equals(PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID)) - .count() > 0) { - return null; - } - - return new ObjectMapper().writeValueAsString(relationList.get(0)); - - }, Encoders.STRING()) - .filter(Objects::nonNull) - .map( - (MapFunction) r -> new ObjectMapper().readValue(r, Relation.class), - Encoders.bean(Relation.class)) - .write() - .mode(SaveMode.Append) - .option("compression", "gzip") - .json(newRelationPath); + if (relationList + .stream() + .filter( + rel -> !rel + .getDataInfo() + .getProvenanceaction() + .getClassid() + .equals(PROPAGATION_RELATION_RESULT_ORGANIZATION_SEM_REL_CLASS_ID) + && !rel + .getDataInfo() + .getProvenanceaction() + .getClassid() + .equals(PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID)) + .count() > 0) { + return null; + } + return new ObjectMapper().writeValueAsString(relationList.get(0)); + }, Encoders.STRING()) + .filter(Objects::nonNull) + .map( + (MapFunction) r -> new ObjectMapper().readValue(r, Relation.class), + Encoders.bean(Relation.class)) + .write() + .mode(SaveMode.Append) + .option("compression", "gzip") + .json(newRelationPath); } @@ -175,7 +176,7 @@ public class StepActions implements Serializable { String leavesPath, String chldParentOrgPath, String entityOrgPath, - String semantics) { + String semantics) { Dataset childParent = readPath(spark, chldParentOrgPath, KeyValueSet.class); Dataset entityOrg = readPath(spark, entityOrgPath, KeyValueSet.class); @@ -202,7 +203,6 @@ public class StepActions implements Serializable { "GROUP BY entityId") .as(Encoders.bean(KeyValueSet.class)); - // create new relations from entity to organization for each entity linked to a leaf return resultParent .flatMap( @@ -213,13 +213,11 @@ public class StepActions implements Serializable { orgId -> getRelation( v.getKey(), orgId, - semantics)) + semantics)) .collect(Collectors.toList()) .iterator(), Encoders.bean(Relation.class)); } - - } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java index f29e8d24a..7c9c2b97b 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/PrepareInfoJobTest.java @@ -77,7 +77,7 @@ public class PrepareInfoJobTest { "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -223,7 +223,7 @@ public class PrepareInfoJobTest { "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -344,7 +344,7 @@ public class PrepareInfoJobTest { "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -365,26 +365,26 @@ public class PrepareInfoJobTest { public void relationProjectTest() throws Exception { PrepareInfo - .main( - new String[] { - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-graphPath", getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest") - .getPath(), - "-hive_metastore_uris", "", - "-leavesPath", workingDir.toString() + "/currentIteration/", - "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", - "-childParentPath", workingDir.toString() + "/childParentOrg/", - "-relationPath", workingDir.toString() + "/relation" + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-graphPath", getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest") + .getPath(), + "-hive_metastore_uris", "", + "-leavesPath", workingDir.toString() + "/currentIteration/", + "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-childParentPath", workingDir.toString() + "/childParentOrg/", + "-relationPath", workingDir.toString() + "/relation" - }); + }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/relation/project") - .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); + .textFile(workingDir.toString() + "/relation/project") + .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); Dataset verificationDs = spark.createDataset(tmp.rdd(), Encoders.bean(Relation.class)); @@ -406,7 +406,7 @@ public class PrepareInfoJobTest { "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -531,134 +531,134 @@ public class PrepareInfoJobTest { public void projectOrganizationTest1() throws Exception { PrepareInfo - .main( - new String[] { - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-graphPath", getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest") - .getPath(), - "-hive_metastore_uris", "", - "-leavesPath", workingDir.toString() + "/currentIteration/", - "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", - "-childParentPath", workingDir.toString() + "/childParentOrg/", - "-relationPath", workingDir.toString() + "/relation" + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-graphPath", getClass() + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/projectorganizationtest") + .getPath(), + "-hive_metastore_uris", "", + "-leavesPath", workingDir.toString() + "/currentIteration/", + "-resultOrgPath", workingDir.toString() + "/resultOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-childParentPath", workingDir.toString() + "/childParentOrg/", + "-relationPath", workingDir.toString() + "/relation" - }); + }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/projectOrganization/") - .map(item -> OBJECT_MAPPER.readValue(item, KeyValueSet.class)); + .textFile(workingDir.toString() + "/projectOrganization/") + .map(item -> OBJECT_MAPPER.readValue(item, KeyValueSet.class)); Dataset verificationDs = spark.createDataset(tmp.rdd(), Encoders.bean(KeyValueSet.class)); Assertions.assertEquals(5, verificationDs.count()); Assertions - .assertEquals( - 2, verificationDs - .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") - .collectAsList() - .get(0) - .getValueSet() - .size()); + .assertEquals( + 2, verificationDs + .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") + .collectAsList() + .get(0) + .getValueSet() + .size()); Assertions - .assertTrue( - verificationDs - .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") - .collectAsList() - .get(0) - .getValueSet() - .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions - .assertTrue( - verificationDs - .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") - .collectAsList() - .get(0) - .getValueSet() - .contains("20|pippo_wf_001::2899e571609779168222fdeb59cb916d")); + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|pippo_wf_001::2899e571609779168222fdeb59cb916d")); Assertions - .assertEquals( - 2, verificationDs - .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") - .collectAsList() - .get(0) - .getValueSet() - .size()); + .assertEquals( + 2, verificationDs + .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") + .collectAsList() + .get(0) + .getValueSet() + .size()); Assertions - .assertTrue( - verificationDs - .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") - .collectAsList() - .get(0) - .getValueSet() - .contains("20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0")); + .assertTrue( + verificationDs + .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|doajarticles::396262ee936f3d3e26ff0e60bea6cae0")); Assertions - .assertTrue( - verificationDs - .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") - .collectAsList() - .get(0) - .getValueSet() - .contains("20|pippo_wf_001::2899e571609779168222fdeb59cb916d")); + .assertTrue( + verificationDs + .filter("key = '40|dedup_wf_001::2899e571609779168222fdeb59cb916d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|pippo_wf_001::2899e571609779168222fdeb59cb916d")); Assertions - .assertEquals( - 1, verificationDs - .filter("key = '40|doajarticles::03748bcb5d754c951efec9700e18a56d'") - .collectAsList() - .get(0) - .getValueSet() - .size()); + .assertEquals( + 1, verificationDs + .filter("key = '40|doajarticles::03748bcb5d754c951efec9700e18a56d'") + .collectAsList() + .get(0) + .getValueSet() + .size()); Assertions - .assertTrue( - verificationDs - .filter("key = '40|doajarticles::03748bcb5d754c951efec9700e18a56d'") - .collectAsList() - .get(0) - .getValueSet() - .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::03748bcb5d754c951efec9700e18a56d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions - .assertEquals( - 1, verificationDs - .filter("key = '40|openaire____::ec653e804967133b9436fdd30d3ff51d'") - .collectAsList() - .get(0) - .getValueSet() - .size()); + .assertEquals( + 1, verificationDs + .filter("key = '40|openaire____::ec653e804967133b9436fdd30d3ff51d'") + .collectAsList() + .get(0) + .getValueSet() + .size()); Assertions - .assertTrue( - verificationDs - .filter("key = '40|openaire____::ec653e804967133b9436fdd30d3ff51d'") - .collectAsList() - .get(0) - .getValueSet() - .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + .assertTrue( + verificationDs + .filter("key = '40|openaire____::ec653e804967133b9436fdd30d3ff51d'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); Assertions - .assertEquals( - 1, verificationDs - .filter("key = '40|doajarticles::1cae0b82b56ccd97c2db1f698def7074'") - .collectAsList() - .get(0) - .getValueSet() - .size()); + .assertEquals( + 1, verificationDs + .filter("key = '40|doajarticles::1cae0b82b56ccd97c2db1f698def7074'") + .collectAsList() + .get(0) + .getValueSet() + .size()); Assertions - .assertTrue( - verificationDs - .filter("key = '40|doajarticles::1cae0b82b56ccd97c2db1f698def7074'") - .collectAsList() - .get(0) - .getValueSet() - .contains("20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1")); + .assertTrue( + verificationDs + .filter("key = '40|doajarticles::1cae0b82b56ccd97c2db1f698def7074'") + .collectAsList() + .get(0) + .getValueSet() + .contains("20|opendoar____::a5fcb8eb25ebd6f7cd219e0fa1e6ddc1")); verificationDs - .foreach((ForeachFunction) v -> System.out.println(OBJECT_MAPPER.writeValueAsString(v))); + .foreach((ForeachFunction) v -> System.out.println(OBJECT_MAPPER.writeValueAsString(v))); } @@ -676,7 +676,7 @@ public class PrepareInfoJobTest { "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" @@ -704,7 +704,7 @@ public class PrepareInfoJobTest { "-hive_metastore_uris", "", "-leavesPath", workingDir.toString() + "/currentIteration/", "-resultOrgPath", workingDir.toString() + "/resultOrganization/", - "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", + "-projectOrganizationPath", workingDir.toString() + "/projectOrganization/", "-childParentPath", workingDir.toString() + "/childParentOrg/", "-relationPath", workingDir.toString() + "/relation" diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java index eb4ade0da..2e75c75ad 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/SparkJobTest.java @@ -85,9 +85,9 @@ public class SparkJobTest { .getPath(); final String projectOrgPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") + .getPath(); readPath(spark, leavesPath, Leaves.class) .write() @@ -100,9 +100,9 @@ public class SparkJobTest { .json(workingDir.toString() + "/orgsInput"); readPath(spark, projectOrgPath, KeyValueSet.class) - .write() - .option("compression", "gzip") - .json(workingDir.toString() + "/projectInput"); + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/projectInput"); SparkResultToOrganizationFromSemRel @@ -114,7 +114,7 @@ public class SparkJobTest { "-outputPath", workingDir.toString() + "/finalrelation", "-leavesPath", workingDir.toString() + "/leavesInput", "-resultOrgPath", workingDir.toString() + "/orgsInput", - "-projectOrganizationPath", workingDir.toString() + "/projectInput", + "-projectOrganizationPath", workingDir.toString() + "/projectInput", "-childParentPath", childParentPath, "-workingDir", workingDir.toString() }); @@ -161,19 +161,24 @@ public class SparkJobTest { .foreach(r -> Assertions.assertEquals(ModelConstants.HAS_AUTHOR_INSTITUTION, r.getRelClass())); Assertions .assertEquals( - 2, result.filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + 2, + result.filter(r -> r.getSource().equals("50|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); Assertions .assertEquals( - 3, result.filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + 3, + result.filter(r -> r.getSource().equals("50|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); Assertions .assertEquals( - 2, result.filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + 2, + result.filter(r -> r.getSource().equals("50|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); Assertions .assertEquals( - 1, result.filter(r -> r.getSource().equals("50|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + 1, + result.filter(r -> r.getSource().equals("50|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); Assertions .assertEquals( - 1, result.filter(r -> r.getSource().equals("50|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + 1, + result.filter(r -> r.getSource().equals("50|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); Assertions.assertEquals(9, result.filter(r -> r.getSource().substring(0, 3).equals("20|")).count()); result @@ -181,19 +186,24 @@ public class SparkJobTest { .foreach(r -> Assertions.assertEquals(ModelConstants.IS_AUTHOR_INSTITUTION_OF, r.getRelClass())); Assertions .assertEquals( - 1, result.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + 1, + result.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); Assertions .assertEquals( - 1, result.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + 1, + result.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); Assertions .assertEquals( - 2, result.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + 2, + result.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); Assertions .assertEquals( - 2, result.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + 2, + result.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); Assertions .assertEquals( - 3, result.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + 3, + result.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); Assertions .assertTrue( @@ -336,332 +346,343 @@ public class SparkJobTest { public void completeProjectExecution() throws Exception { final String graphPath = getClass() - .getResource("/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph") - .getPath(); + .getResource("/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph") + .getPath(); final String leavesPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") + .getPath(); final String childParentPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") + .getPath(); final String resultOrgPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") + .getPath(); final String projectOrgPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") + .getPath(); readPath(spark, leavesPath, Leaves.class) - .write() - .option("compression", "gzip") - .json(workingDir.toString() + "/leavesInput"); + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/leavesInput"); readPath(spark, resultOrgPath, KeyValueSet.class) - .write() - .option("compression", "gzip") - .json(workingDir.toString() + "/orgsInput"); + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/orgsInput"); readPath(spark, projectOrgPath, KeyValueSet.class) - .write() - .option("compression", "gzip") - .json(workingDir.toString() + "/projectInput"); + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/projectInput"); SparkResultToOrganizationFromSemRel - .main( - new String[] { - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-relationPath", graphPath, - "-hive_metastore_uris", "", - "-outputPath", workingDir.toString() + "/finalrelation", - "-leavesPath", workingDir.toString() + "/leavesInput", - "-resultOrgPath", workingDir.toString() + "/orgsInput", - "-projectOrganizationPath", workingDir.toString() + "/projectInput", - "-childParentPath", childParentPath, - "-workingDir", workingDir.toString() - }); + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-relationPath", graphPath, + "-hive_metastore_uris", "", + "-outputPath", workingDir.toString() + "/finalrelation", + "-leavesPath", workingDir.toString() + "/leavesInput", + "-resultOrgPath", workingDir.toString() + "/orgsInput", + "-projectOrganizationPath", workingDir.toString() + "/projectInput", + "-childParentPath", childParentPath, + "-workingDir", workingDir.toString() + }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD temp = sc - .textFile(workingDir.toString() + "/finalrelation") - .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); + .textFile(workingDir.toString() + "/finalrelation") + .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); Assertions.assertEquals(36, temp.count()); - JavaRDD project = temp.filter(r -> r.getSource().startsWith("40|") || r.getTarget().startsWith("40|")); + JavaRDD project = temp + .filter(r -> r.getSource().startsWith("40|") || r.getTarget().startsWith("40|")); Assertions.assertEquals(18, project.count()); project.foreach(r -> Assertions.assertEquals(ModelConstants.PARTICIPATION, r.getSubRelType())); project.foreach(r -> Assertions.assertEquals(ModelConstants.PROJECT_ORGANIZATION, r.getRelType())); project - .foreach( - r -> Assertions - .assertEquals( - PropagationConstant.PROPAGATION_DATA_INFO_TYPE, r.getDataInfo().getInferenceprovenance())); + .foreach( + r -> Assertions + .assertEquals( + PropagationConstant.PROPAGATION_DATA_INFO_TYPE, r.getDataInfo().getInferenceprovenance())); project - .foreach( - r -> Assertions - .assertEquals( - PropagationConstant.PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID, - r.getDataInfo().getProvenanceaction().getClassid())); + .foreach( + r -> Assertions + .assertEquals( + PropagationConstant.PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_ID, + r.getDataInfo().getProvenanceaction().getClassid())); project - .foreach( - r -> Assertions - .assertEquals( - PropagationConstant.PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_NAME, - r.getDataInfo().getProvenanceaction().getClassname())); + .foreach( + r -> Assertions + .assertEquals( + PropagationConstant.PROPAGATION_RELATION_PROJECT_ORGANIZATION_SEM_REL_CLASS_NAME, + r.getDataInfo().getProvenanceaction().getClassname())); project - .foreach( - r -> Assertions - .assertEquals( - "0.85", - r.getDataInfo().getTrust())); + .foreach( + r -> Assertions + .assertEquals( + "0.85", + r.getDataInfo().getTrust())); Assertions.assertEquals(9, project.filter(r -> r.getSource().substring(0, 3).equals("40|")).count()); project - .filter(r -> r.getSource().substring(0, 3).equals("40|")) - .foreach(r -> Assertions.assertEquals(ModelConstants.HAS_PARTICIPANT, r.getRelClass())); + .filter(r -> r.getSource().substring(0, 3).equals("40|")) + .foreach(r -> Assertions.assertEquals(ModelConstants.HAS_PARTICIPANT, r.getRelClass())); Assertions - .assertEquals( - 2, project.filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + .assertEquals( + 2, + project.filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); Assertions - .assertEquals( - 3, project.filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + .assertEquals( + 3, + project.filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); Assertions - .assertEquals( - 2, project.filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + .assertEquals( + 2, + project.filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); Assertions - .assertEquals( - 1, project.filter(r -> r.getSource().equals("40|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + .assertEquals( + 1, + project.filter(r -> r.getSource().equals("40|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); Assertions - .assertEquals( - 1, project.filter(r -> r.getSource().equals("40|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + .assertEquals( + 1, + project.filter(r -> r.getSource().equals("40|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); Assertions.assertEquals(9, project.filter(r -> r.getSource().substring(0, 3).equals("20|")).count()); project - .filter(r -> r.getSource().substring(0, 3).equals("20|")) - .foreach(r -> Assertions.assertEquals(ModelConstants.IS_PARTICIPANT, r.getRelClass())); + .filter(r -> r.getSource().substring(0, 3).equals("20|")) + .foreach(r -> Assertions.assertEquals(ModelConstants.IS_PARTICIPANT, r.getRelClass())); Assertions - .assertEquals( - 1, project.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); + .assertEquals( + 1, + project.filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")).count()); Assertions - .assertEquals( - 1, project.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); + .assertEquals( + 1, + project.filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")).count()); Assertions - .assertEquals( - 2, project.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); + .assertEquals( + 2, + project.filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")).count()); Assertions - .assertEquals( - 2, project.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); + .assertEquals( + 2, + project.filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")).count()); Assertions - .assertEquals( - 3, project.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); + .assertEquals( + 3, + project.filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")).count()); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) - .map(r -> r.getTarget()) - .collect() - .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) - .map(r -> r.getTarget()) - .collect() - .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) + .map(r -> r.getTarget()) + .collect() + .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) - .map(r -> r.getTarget()) - .collect() - .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) - .map(r -> r.getTarget()) - .collect() - .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) - .map(r -> r.getTarget()) - .collect() - .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) - .map(r -> r.getTarget()) - .collect() - .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) - .map(r -> r.getTarget()) - .collect() - .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|openaire____::ec653e804967133b9436fdd30d3ff51d")) - .map(r -> r.getTarget()) - .collect() - .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|openaire____::ec653e804967133b9436fdd30d3ff51d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|openaire____::ec653e804967133b9436fdd30d3ff51d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("40|doajarticles::03748bcb5d754c951efec9700e18a56d")) - .map(r -> r.getTarget()) - .collect() - .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("40|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("20|doajarticles::03748bcb5d754c951efec9700e18a56d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) - .map(r -> r.getTarget()) - .collect() - .contains("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) - .map(r -> r.getTarget()) - .collect() - .contains("40|openaire____::ec653e804967133b9436fdd30d3ff51d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|openaire____::ec653e804967133b9436fdd30d3ff51d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|openaire____::ec653e804967133b9436fdd30d3ff51d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) - .map(r -> r.getTarget()) - .collect() - .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) - .map(r -> r.getTarget()) - .collect() - .contains("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) - .map(r -> r.getTarget()) - .collect() - .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) - .map(r -> r.getTarget()) - .collect() - .contains("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::2baa9032dc058d3c8ff780c426b0c19f")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) - .map(r -> r.getTarget()) - .collect() - .contains("40|doajarticles::03748bcb5d754c951efec9700e18a56d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::03748bcb5d754c951efec9700e18a56d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::03748bcb5d754c951efec9700e18a56d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")) - .map(r -> r.getTarget()) - .collect() - .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|dedup_wf_001::2899e571609779168222fdeb59cb916d")) + .map(r -> r.getTarget()) + .collect() + .contains("40|dedup_wf_001::2899e571609779168222fdeb59cb916d")); Assertions - .assertTrue( - project - .filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) - .map(r -> r.getTarget()) - .collect() - .contains("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); + .assertTrue( + project + .filter(r -> r.getSource().equals("20|doajarticles::1cae0b82b56ccd97c2db1f698def7074")) + .map(r -> r.getTarget()) + .collect() + .contains("40|doajarticles::1cae0b82b56ccd97c2db1f698def7074")); } @Test public void singleIterationExecution() throws Exception { final String graphPath = getClass() - .getResource("/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph") - .getPath(); + .getResource("/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/graph") + .getPath(); final String leavesPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/currentIteration/") + .getPath(); final String childParentPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/childParentOrg/") + .getPath(); final String resultOrgPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") + .getPath(); final String projectOrgPath = getClass() - .getResource( - "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/projectOrganization/") + .getPath(); readPath(spark, leavesPath, Leaves.class) - .write() - .option("compression", "gzip") - .json(workingDir.toString() + "/leavesInput"); + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/leavesInput"); readPath(spark, resultOrgPath, KeyValueSet.class) - .write() - .option("compression", "gzip") - .json(workingDir.toString() + "/orgsInput"); + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/orgsInput"); readPath(spark, projectOrgPath, KeyValueSet.class) - .write() - .option("compression", "gzip") - .json(workingDir.toString() + "/projectInput"); + .write() + .option("compression", "gzip") + .json(workingDir.toString() + "/projectInput"); SparkResultToOrganizationFromSemRel - .main( - new String[] { - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-relationPath", graphPath, - "-hive_metastore_uris", "", - "-outputPath", workingDir.toString() + "/finalrelation", - "-leavesPath", workingDir.toString() + "/leavesInput", - "-resultOrgPath", workingDir.toString() + "/orgsInput", - "-projectOrganizationPath", workingDir.toString() + "/projectInput", - "-childParentPath", childParentPath, - "-workingDir", workingDir.toString(), - "-iterations", "1" - }); + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-relationPath", graphPath, + "-hive_metastore_uris", "", + "-outputPath", workingDir.toString() + "/finalrelation", + "-leavesPath", workingDir.toString() + "/leavesInput", + "-resultOrgPath", workingDir.toString() + "/orgsInput", + "-projectOrganizationPath", workingDir.toString() + "/projectInput", + "-childParentPath", childParentPath, + "-workingDir", workingDir.toString(), + "-iterations", "1" + }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD temp = sc - .textFile(workingDir.toString() + "/finalrelation") - .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); + .textFile(workingDir.toString() + "/finalrelation") + .map(item -> OBJECT_MAPPER.readValue(item, Relation.class)); Assertions.assertEquals(16, temp.count()); diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java index 7a71240b2..64339e3b7 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/entitytoorganizationfromsemrel/StepActionsTest.java @@ -87,7 +87,8 @@ public class StepActionsTest { getClass() .getResource( "/eu/dnetlib/dhp/entitytoorganizationfromsemrel/execstep/resultOrganization/") - .getPath(), ModelConstants.HAS_AUTHOR_INSTITUTION); + .getPath(), + ModelConstants.HAS_AUTHOR_INSTITUTION); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); From e1409ffe8057dd740f9123e13c1a07e5b3cebff3 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 12 Jun 2023 09:47:45 +0200 Subject: [PATCH 8/8] update sql query to return distinct pids --- .../dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql index 133b817d2..cba08d466 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql @@ -15,7 +15,10 @@ SELECT 'OpenOrgs Database' AS collectedfromname, o.country || '@@@dnet:countries' AS country, 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, - array_remove(array_cat(array_agg(DISTINCT i.otherid || '###' || i.type || '@@@dnet:pid_types'), array_agg(DISTINCT idup.otherid || '###' || idup.type || '@@@dnet:pid_types')), NULL) AS pid, + ARRAY(SELECT DISTINCT pid FROM unnest(array_cat( + array_agg(DISTINCT i.otherid || '###' || i.type || '@@@dnet:pid_types'), + array_agg(DISTINCT idup.otherid || '###' || idup.type || '@@@dnet:pid_types') + )) as t(pid) where pid IS NOT NULL), (array_remove(array_cat(ARRAY[o.ec_legalbody], array_agg(od.ec_legalbody)), NULL))[1] AS eclegalbody, (array_remove(array_cat(ARRAY[o.ec_legalperson], array_agg(od.ec_legalperson)), NULL))[1] AS eclegalperson, (array_remove(array_cat(ARRAY[o.ec_nonprofit], array_agg(od.ec_nonprofit)), NULL))[1] AS ecnonprofit,